def process_bulk_add_domain(request, formdict):
"""
Performs the bulk add of domains by parsing the request data. Batches
some data into a cache object for performance by reducing large
amounts of single database queries.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param formdict: The form representing the bulk uploaded data.
:type formdict: dict
:returns: :class:`django.http.HttpResponse`
"""
domain_names = []
ip_addresses = []
cached_domain_results = {}
cached_ip_results = {}
cleanedRowsData = convert_handsontable_to_rows(request)
for rowData in cleanedRowsData:
if rowData != None:
if rowData.get(form_consts.Domain.DOMAIN_NAME) != None:
domain = rowData.get(
form_consts.Domain.DOMAIN_NAME).strip().lower()
(root_domain, full_domain,
error) = get_valid_root_domain(domain)
domain_names.append(full_domain)
if domain != root_domain:
domain_names.append(root_domain)
if rowData.get(form_consts.Domain.IP_ADDRESS) != None:
ip_addr = rowData.get(form_consts.Domain.IP_ADDRESS)
ip_type = rowData.get(form_consts.Domain.IP_TYPE)
(ip_addr, error) = validate_and_normalize_ip(ip_addr, ip_type)
ip_addresses.append(ip_addr)
domain_results = Domain.objects(domain__in=domain_names)
ip_results = IP.objects(ip__in=ip_addresses)
for domain_result in domain_results:
cached_domain_results[domain_result.domain] = domain_result
for ip_result in ip_results:
cached_ip_results[ip_result.ip] = ip_result
cache = {
form_consts.Domain.CACHED_RESULTS: cached_domain_results,
form_consts.IP.CACHED_RESULTS: cached_ip_results,
'cleaned_rows_data': cleanedRowsData
}
response = parse_bulk_upload(request, parse_row_to_bound_domain_form,
add_new_domain_via_bulk, formdict, cache)
return response
def process_bulk_add_domain(request, formdict):
"""
Performs the bulk add of domains by parsing the request data. Batches
some data into a cache object for performance by reducing large
amounts of single database queries.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param formdict: The form representing the bulk uploaded data.
:type formdict: dict
:returns: :class:`django.http.HttpResponse`
"""
domain_names = []
ip_addresses = []
cached_domain_results = {}
cached_ip_results = {}
cleanedRowsData = convert_handsontable_to_rows(request)
for rowData in cleanedRowsData:
if rowData != None:
if rowData.get(form_consts.Domain.DOMAIN_NAME) != None:
domain = rowData.get(form_consts.Domain.DOMAIN_NAME).strip().lower()
(root_domain, full_domain, error) = get_valid_root_domain(domain)
domain_names.append(full_domain)
if domain != root_domain:
domain_names.append(root_domain)
if rowData.get(form_consts.Domain.IP_ADDRESS) != None:
ip_addr = rowData.get(form_consts.Domain.IP_ADDRESS)
ip_type = rowData.get(form_consts.Domain.IP_TYPE)
(ip_addr, error) = validate_and_normalize_ip(ip_addr, ip_type)
ip_addresses.append(ip_addr)
domain_results = Domain.objects(domain__in=domain_names)
ip_results = IP.objects(ip__in=ip_addresses)
for domain_result in domain_results:
cached_domain_results[domain_result.domain] = domain_result
for ip_result in ip_results:
cached_ip_results[ip_result.ip] = ip_result
cache = {
form_consts.Domain.CACHED_RESULTS: cached_domain_results,
form_consts.IP.CACHED_RESULTS: cached_ip_results,
"cleaned_rows_data": cleanedRowsData,
}
response = parse_bulk_upload(request, parse_row_to_bound_domain_form, add_new_domain_via_bulk, formdict, cache)
return response
def add_new_domain(data,
request,
errors,
rowData=None,
is_validate_only=False,
cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: tuple (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
domain = data['domain']
add_ip = data.get('add_ip')
ip = data.get('ip')
ip_type = data.get('ip_type')
if add_ip:
error = validate_and_normalize_ip(ip, ip_type)[1]
if error:
errors.append(error)
if is_validate_only:
error = get_valid_root_domain(domain)[2]
if error:
errors.append(error)
# check for duplicate domains
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail',
args=[domain])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
result_cache[domain.lower()] = True
elif not errors:
username = request.user.username
reference = data.get('domain_reference')
source_name = data.get('domain_source')
method = data.get('domain_method')
source = [
create_embedded_source(source_name,
reference=reference,
method=method,
analyst=username)
]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [
EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)
]
else:
campaign = []
retVal = upsert_domain(domain,
source,
username,
campaign,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not retVal['success']:
errors.append(retVal.get('message'))
retVal['message'] = ""
else:
new_domain = retVal['object']
ip_result = {}
if add_ip:
if data.get('same_source'):
ip_source = source_name
ip_method = method
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(rel_item=new_ip,
rel_type='Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
new_ip.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail',
args=[domain])
if retVal['is_domain_new'] == True:
retVal['message'] = (
'Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, domain))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' %
(resp_url, domain))
retVal['message'] = message
retVal[form_consts.Status.
STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_tlo
# If we have an IP object, add an indicator for that.
if ip_result.get('success'):
ip = ip_result['object']
result = create_indicator_from_tlo('IP',
ip,
username,
ip_source,
add_domain=False)
ip_ind = result.get('indicator')
if not result['success']:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_tlo('Domain',
new_domain,
username,
source_name,
add_domain=False)
if not result['success']:
errors.append(result['message'])
elif ip_result.get('success') and ip_ind:
forge_relationship(left_class=result['indicator'],
right_class=ip_ind,
rel_type='Resolved_To',
analyst=username)
result = True
# This block validates, and may also add, objects to the Domain
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(domain, cache)
for object_row_counter, objectData in enumerate(
objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(
objectData, "Domain", current_domain.id)
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
current_domain = None
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
(obj_result, errors,
obj_retVal) = validate_and_add_new_handler_object(
None,
objectDict,
request,
errors,
object_row_counter,
is_validate_only=is_validate_only,
cache=cache,
obj=current_domain)
if not obj_result:
retVal['success'] = False
return result, errors, retVal
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: tuple (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
domain = data['domain']
add_ip = data.get('add_ip')
ip = data.get('ip')
ip_type = data.get('ip_type')
if add_ip:
error = validate_and_normalize_ip(ip, ip_type)[1]
if error:
errors.append(error)
if is_validate_only:
error = get_valid_root_domain(domain)[2]
if error:
errors.append(error)
# check for duplicate domains
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
result_cache[domain.lower()] = True
elif not errors:
username = request.user.username
reference = data.get('domain_reference')
source_name = data.get('domain_source')
method = data.get('domain_method')
source = [create_embedded_source(source_name, reference=reference,
method=method, analyst=username)]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)]
else:
campaign = []
retVal = upsert_domain(domain, source, username, campaign,
bucket_list=bucket_list, ticket=ticket, cache=cache)
if not retVal['success']:
errors.append(retVal.get('message'))
retVal['message'] = ""
else:
new_domain = retVal['object']
ip_result = {}
if add_ip:
if data.get('same_source'):
ip_source = source_name
ip_method = method
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(new_ip,
'Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
if retVal['is_domain_new'] == True:
retVal['message'] = ('Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, domain))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_tlo
# If we have an IP object, add an indicator for that.
if ip_result.get('success'):
ip = ip_result['object']
result = create_indicator_from_tlo('IP',
ip,
username,
ip_source,
add_domain=False)
ip_ind = result.get('indicator')
if not result['success']:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_tlo('Domain',
new_domain,
username,
source_name,
add_domain=False)
if not result['success']:
errors.append(result['message'])
elif ip_result.get('success') and ip_ind:
forge_relationship(left_class=result['indicator'],
right_class=ip_ind,
rel_type='Resolved_To',
analyst=username)
result = True
# This block validates, and may also add, objects to the Domain
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(domain, cache)
for object_row_counter, objectData in enumerate(objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(objectData,
"Domain",
current_domain.id)
else:
objectDict = object_array_to_dict(objectData,
"Domain",
"")
current_domain = None;
else:
objectDict = object_array_to_dict(objectData,
"Domain",
"")
(obj_result,
errors,
obj_retVal) = validate_and_add_new_handler_object(
None, objectDict, request, errors, object_row_counter,
is_validate_only=is_validate_only,
cache=cache, obj=current_domain)
if not obj_result:
retVal['success'] = False
return result, errors, retVal
def validate_indicator_value(value, ind_type):
"""
Check that a given value is valid for a particular Indicator type.
:param value: The value to be validated
:type value: str
:param ind_type: The indicator type to validate against
:type ind_type: str
:returns: tuple: (Valid value, Error message)
"""
value = value.strip()
domain = ""
# URL
if ind_type == IndicatorTypes.URI:
if "://" not in value.split(".")[0]:
return ("", "URI must contain protocol " "prefix (e.g. http://, https://, ftp://) ")
domain_or_ip = urlparse.urlparse(value).hostname
try:
validate_ipv46_address(domain_or_ip)
return (value, "")
except DjangoValidationError:
domain = domain_or_ip
# Email address
if ind_type in (
IndicatorTypes.EMAIL_ADDRESS,
IndicatorTypes.EMAIL_FROM,
IndicatorTypes.EMAIL_REPLY_TO,
IndicatorTypes.EMAIL_SENDER,
):
if "@" not in value:
return ("", "Email address must contain an '@'")
domain_or_ip = value.split("@")[-1]
if domain_or_ip[0] == "[" and domain_or_ip[-1] == "]":
try:
validate_ipv46_address(domain_or_ip[1:-1])
return (value, "")
except DjangoValidationError:
return ("", "Email address does not contain a valid IP")
else:
domain = domain_or_ip
# IPs
if ind_type in IPTypes.values():
(ip_address, error) = validate_and_normalize_ip(value, ind_type)
if error:
return ("", error)
else:
return (ip_address, "")
# Domains
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI) or domain:
(root, domain, error) = get_valid_root_domain(domain or value)
if error:
return ("", error)
else:
return (value, "")
return (value, "")
