def parse_row_to_bound_object_form(request, rowData, cache):
"""
Parse a row from mass object upload into an AddObjectForm.
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param rowData: The row data.
:type rowData: dict
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: :class:`crits.objects.forms.AddObjectForm`
"""
bound_form = None
# TODO fix the hardcoded strings and conversion of types
# TODO Add common method to convert data to string
object_type = rowData.get(form_consts.Object.OBJECT_TYPE, "")
value = rowData.get(form_consts.Object.VALUE, "")
source = rowData.get(form_consts.Object.SOURCE, "")
method = rowData.get(form_consts.Object.METHOD, "")
reference = rowData.get(form_consts.Object.REFERENCE, "")
otype = rowData.get(form_consts.Object.PARENT_OBJECT_TYPE, "")
oid = rowData.get(form_consts.Object.PARENT_OBJECT_ID, "")
is_add_indicator = convert_string_to_bool(
rowData.get(form_consts.Object.ADD_INDICATOR, "False"))
all_obj_type_choices = cache.get("object_types")
if all_obj_type_choices == None:
all_obj_type_choices = [(c[0], c[0], {
'datatype': c[1].keys()[0],
'datatype_value': c[1].values()[0]
}) for c in get_object_types(False)]
cache["object_types"] = all_obj_type_choices
data = {
'object_type': object_type,
'value': value,
'source': source,
'method': method,
'reference': reference,
'otype': otype,
'oid': oid,
'add_indicator': is_add_indicator
}
bound_form = cache.get("object_form")
if bound_form == None:
bound_form = AddObjectForm(request.user, all_obj_type_choices, data)
cache['object_form'] = bound_form
else:
bound_form.data = data
bound_form.full_clean()
return bound_form
def parse_row_to_bound_object_form(request, rowData, cache):
"""
Parse a row from mass object upload into an AddObjectForm.
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param rowData: The row data.
:type rowData: dict
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: :class:`crits.objects.forms.AddObjectForm`
"""
bound_form = None
# TODO fix the hardcoded strings and conversion of types
# TODO Add common method to convert data to string
object_type = rowData.get(form_consts.Object.OBJECT_TYPE, "")
value = rowData.get(form_consts.Object.VALUE, "")
source = rowData.get(form_consts.Object.SOURCE, "")
method = rowData.get(form_consts.Object.METHOD, "")
reference = rowData.get(form_consts.Object.REFERENCE, "")
otype = rowData.get(form_consts.Object.PARENT_OBJECT_TYPE, "")
oid = rowData.get(form_consts.Object.PARENT_OBJECT_ID, "")
is_add_indicator = convert_string_to_bool(rowData.get(form_consts.Object.ADD_INDICATOR, "False"))
all_obj_type_choices = cache.get("object_types")
if all_obj_type_choices == None:
all_obj_type_choices = [(c[0],
c[0],
{'datatype':c[1].keys()[0],
'datatype_value':c[1].values()[0]}
) for c in get_object_types(False)]
cache["object_types"] = all_obj_type_choices
data = {
'object_type': object_type,
'value': value,
'source': source,
'method': method,
'reference': reference,
'otype': otype,
'oid': oid,
'add_indicator': is_add_indicator
}
bound_form = cache.get("object_form")
if bound_form == None:
bound_form = AddObjectForm(request.user, all_obj_type_choices, data)
cache['object_form'] = bound_form
else:
bound_form.data = data
bound_form.full_clean()
return bound_form
def parse_row_to_bound_ip_form(request, rowData, cache):
"""
Parse a row out of mass object adder into the
:class:`crits.ips.forms.AddIPForm`.
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param rowData: The data for that row.
:type rowData: dict
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: :class:`crits.ips.forms.AddIPForm`.
"""
# TODO Add common method to convert data to string
ip = rowData.get(form_consts.IP.IP_ADDRESS, "")
ip_type = rowData.get(form_consts.IP.IP_TYPE, "")
# analyst = rowData.get(form_consts.IP.ANALYST, "")
analyst = request.user
campaign = rowData.get(form_consts.IP.CAMPAIGN, "")
confidence = rowData.get(form_consts.IP.CAMPAIGN_CONFIDENCE, "")
source = rowData.get(form_consts.IP.SOURCE, "")
source_method = rowData.get(form_consts.IP.SOURCE_METHOD, "")
source_reference = rowData.get(form_consts.IP.SOURCE_REFERENCE, "")
is_add_indicator = convert_string_to_bool(
rowData.get(form_consts.IP.ADD_INDICATOR, "False"))
indicator_reference = rowData.get(form_consts.IP.INDICATOR_REFERENCE, "")
bucket_list = rowData.get(form_consts.Common.BUCKET_LIST, "")
ticket = rowData.get(form_consts.Common.TICKET, "")
data = {
'ip': ip,
'ip_type': ip_type,
'analyst': analyst,
'campaign': campaign,
'confidence': confidence,
'source': source,
'source_method': source_method,
'source_reference': source_reference,
'add_indicator': is_add_indicator,
'indicator_reference': indicator_reference,
'bucket_list': bucket_list,
'ticket': ticket
}
bound_form = cache.get('ip_form')
if bound_form == None:
bound_form = AddIPForm(request.user, None, data)
cache['ip_form'] = bound_form
else:
bound_form.data = data
bound_form.full_clean()
return bound_form
def parse_row_to_bound_ip_form(request, rowData, cache):
"""
Parse a row out of mass object adder into the
:class:`crits.ips.forms.AddIPForm`.
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param rowData: The data for that row.
:type rowData: dict
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: :class:`crits.ips.forms.AddIPForm`.
"""
# TODO Add common method to convert data to string
ip = rowData.get(form_consts.IP.IP_ADDRESS, "")
ip_type = rowData.get(form_consts.IP.IP_TYPE, "")
# analyst = rowData.get(form_consts.IP.ANALYST, "")
analyst = request.user
campaign = rowData.get(form_consts.IP.CAMPAIGN, "")
confidence = rowData.get(form_consts.IP.CAMPAIGN_CONFIDENCE, "")
source = rowData.get(form_consts.IP.SOURCE, "")
source_method = rowData.get(form_consts.IP.SOURCE_METHOD, "")
source_reference = rowData.get(form_consts.IP.SOURCE_REFERENCE, "")
is_add_indicator = convert_string_to_bool(rowData.get(form_consts.IP.ADD_INDICATOR, "False"))
indicator_reference = rowData.get(form_consts.IP.INDICATOR_REFERENCE, "")
bucket_list = rowData.get(form_consts.Common.BUCKET_LIST, "")
ticket = rowData.get(form_consts.Common.TICKET, "")
data = {
"ip": ip,
"ip_type": ip_type,
"analyst": analyst,
"campaign": campaign,
"confidence": confidence,
"source": source,
"source_method": source_method,
"source_reference": source_reference,
"add_indicator": is_add_indicator,
"indicator_reference": indicator_reference,
"bucket_list": bucket_list,
"ticket": ticket,
}
bound_form = cache.get("ip_form")
if bound_form == None:
bound_form = AddIPForm(request.user, None, data)
cache["ip_form"] = bound_form
else:
bound_form.data = data
bound_form.full_clean()
return bound_form
def parse_row_to_bound_domain_form(request, rowData, cache):
"""
Parse a row in bulk upload into form data that can be used to add a Domain.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param rowData: The objects to add for the Domain.
:type rowData: dict
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: :class:`crits.domains.forms.AddDomainForm`
"""
bound_domain_form = None
# TODO Add common method to convert data to string
domain_name = rowData.get(form_consts.Domain.DOMAIN_NAME, "").strip()
campaign = rowData.get(form_consts.Domain.CAMPAIGN, "")
confidence = rowData.get(form_consts.Domain.CAMPAIGN_CONFIDENCE, "")
domain_source = rowData.get(form_consts.Domain.DOMAIN_SOURCE, "")
domain_method = rowData.get(form_consts.Domain.DOMAIN_METHOD, "")
domain_reference = rowData.get(form_consts.Domain.DOMAIN_REFERENCE, "")
#is_add_ip = convert_string_to_bool(rowData.get(form_consts.Domain.ADD_IP_ADDRESS, ""))
is_add_ip = False
ip = rowData.get(form_consts.Domain.IP_ADDRESS, "")
created = rowData.get(form_consts.Domain.IP_DATE, "")
#is_same_source = convert_string_to_bool(rowData.get(form_consts.Domain.SAME_SOURCE, "False"))
is_same_source = False
ip_source = rowData.get(form_consts.Domain.IP_SOURCE, "")
ip_method = rowData.get(form_consts.Domain.IP_METHOD, "")
ip_reference = rowData.get(form_consts.Domain.IP_REFERENCE, "")
is_add_indicator = convert_string_to_bool(
rowData.get(form_consts.Domain.ADD_INDICATOR, "False"))
bucket_list = rowData.get(form_consts.Common.BUCKET_LIST, "")
ticket = rowData.get(form_consts.Common.TICKET, "")
if (ip or created or ip_source or ip_method or ip_reference):
is_add_ip = True
if is_add_ip == True:
data = {
'domain': domain_name,
'campaign': campaign,
'confidence': confidence,
'domain_source': domain_source,
'domain_method': domain_method,
'domain_reference': domain_reference,
'add_ip': is_add_ip,
'ip': ip,
'created': created,
'same_source': is_same_source,
'ip_source': ip_source,
'ip_method': ip_method,
'ip_reference': ip_reference,
'add_indicators': is_add_indicator,
'bucket_list': bucket_list,
'ticket': ticket
}
bound_domain_form = cache.get("domain_ip_form")
if bound_domain_form == None:
bound_domain_form = AddDomainForm(request.user, data)
cache['domain_ip_form'] = bound_domain_form
else:
bound_domain_form.data = data
else:
data = {
'domain': domain_name,
'campaign': campaign,
'confidence': confidence,
'domain_source': domain_source,
'domain_method': domain_method,
'domain_reference': domain_reference,
'add_ip': is_add_ip,
'bucket_list': bucket_list,
'ticket': ticket
}
bound_domain_form = cache.get("domain_form")
if bound_domain_form == None:
bound_domain_form = AddDomainForm(request.user, data)
cache['domain_form'] = bound_domain_form
else:
bound_domain_form.data = data
if bound_domain_form != None:
bound_domain_form.full_clean()
return bound_domain_form
def parse_bulk_upload(request, parse_row_function, add_new_function, formdict, cache={}):
"""Bulk adds data by parsing an input 2D array serially adding each row.
Args:
request: The Django context which contains information about the
session and the 2D data array for the handsontable bulk add request
parse_row_function: A callback function that will handle parsing
of each row's fields and values and then return the Django
form representation.
add_new_function: A callback function that will handle bulk adding
a single row.
formdict: The dictionary representation of the form
Returns:
Returns an array of dictionaries from the input form. Includes the
following keys in the dictionary:
failedRows: An array with a dictionary of key/value pairs with
information about which rows and why the failure occurred.
The following are the dictionary keys:
row: The row of the failure
col: The column of the failure. If the value is -1
then that means the column where the error occurred
is not determined.
label: The label of the column, if available
message: The reason for the failure
messages: Helpful messages about the bulk add operations
success: A boolean indicating if the bulk add operations were successful
successfulRows: An array with a dictionary of key/value pairs with
information about which rows were successfully bulk added.
The following are the dictionary keys:
row: The row that was successful
secondary: An array of information that can be passed from the
add_new_function() function call back to the caller of this
function. There are no restrictions on the data that can be
passed back. Usually this is used for post processing
analysis of the entire bulk add operation as a whole.
"""
failedRows = []
successfulRows = []
messages = []
secondaryData = []
isFailureDetected = False
is_validate_only = convert_string_to_bool(request.POST['isValidateOnly'])
offset = int(request.POST.get('offset', 0));
if cache.get("cleaned_rows_data"):
cleanedRowsData = cache.get("cleaned_rows_data");
else:
cleanedRowsData = convert_handsontable_to_rows(request);
rowCounter = offset;
processedRows = 0;
for rowData in cleanedRowsData:
rowCounter = rowCounter + 1
# Make sure that the rowData has content, otherwise the client side
# might have done some filtering to ignore rows that have no data.
if(rowData == None):
continue;
try:
bound_form = parse_row_function(request, rowData, cache)
# If bound form passes validation checks then continue adding the item
if bound_form.is_valid():
data = bound_form.cleaned_data
errors = []
retVal = {}
message = ""
(result, errors, retVal) = add_new_function(data, rowData, request, errors, is_validate_only, cache)
processedRows += 1;
#if retVal.get('message'):
# messages.append("At row " + str(rowCounter) + ": " + retVal.get('message'))
if retVal.get('secondary'):
secondaryData.append(retVal.get('secondary'))
# Check to make sure there were no errors
for error in errors:
message += error + '; '
else:
message = message[0:-2] # Remove '; ' from last
if errors:
# If there was an error then use a "col" value of -1, this is needed because
# there is no easy way to indicate the specific column that caused the error.
# The "col": -1 will just indicate to the client side to highlight the entire row.
failedRows.append({'row': rowCounter,
'col': -1,
'message': "At row " + str(rowCounter) + ": " + message});
else:
status = retVal.get(form_consts.Status.STATUS_FIELD, form_consts.Status.SUCCESS);
data = {'row': rowCounter, 's': status};
if retVal.get('warning'):
data['message'] = "At row " + str(rowCounter) + ": " + retVal.get('warning');
elif retVal.get('message'):
data['message'] = "At row " + str(rowCounter) + ": " + retVal.get('message');
successfulRows.append(data);
# ... otherwise validation of the bound form failed and we need to
# populate the response with error information
else:
processedRows += 1;
for name, errorMessages in bound_form.errors.items():
entry = get_field_from_label(name, formdict)
if entry == None:
continue
for message in errorMessages:
failedRows.append({'row': rowCounter,
'col': entry['position'],
'label': entry['label'],
'message': "At (" + str(rowCounter) + ", " + entry['label'] + "): " + message});
except Exception,e:
import traceback
traceback.print_exc()
def parse_row_to_bound_domain_form(request, rowData, cache):
"""
Parse a row in bulk upload into form data that can be used to add a Domain.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param rowData: The objects to add for the Domain.
:type rowData: dict
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: :class:`crits.domains.forms.AddDomainForm`
"""
bound_domain_form = None
# TODO Add common method to convert data to string
domain_name = rowData.get(form_consts.Domain.DOMAIN_NAME, "").strip();
campaign = rowData.get(form_consts.Domain.CAMPAIGN, "")
confidence = rowData.get(form_consts.Domain.CAMPAIGN_CONFIDENCE, "")
domain_source = rowData.get(form_consts.Domain.DOMAIN_SOURCE, "")
domain_method = rowData.get(form_consts.Domain.DOMAIN_METHOD, "")
domain_reference = rowData.get(form_consts.Domain.DOMAIN_REFERENCE, "")
#is_add_ip = convert_string_to_bool(rowData.get(form_consts.Domain.ADD_IP_ADDRESS, ""))
is_add_ip = False
ip = rowData.get(form_consts.Domain.IP_ADDRESS, "")
ip_type = rowData.get(form_consts.Domain.IP_TYPE, "")
created = rowData.get(form_consts.Domain.IP_DATE, "")
#is_same_source = convert_string_to_bool(rowData.get(form_consts.Domain.SAME_SOURCE, "False"))
is_same_source = False
ip_source = rowData.get(form_consts.Domain.IP_SOURCE, "")
ip_method = rowData.get(form_consts.Domain.IP_METHOD, "")
ip_reference = rowData.get(form_consts.Domain.IP_REFERENCE, "")
is_add_indicators = convert_string_to_bool(rowData.get(form_consts.Domain.ADD_INDICATORS, "False"))
bucket_list = rowData.get(form_consts.Common.BUCKET_LIST, "")
ticket = rowData.get(form_consts.Common.TICKET, "")
if(ip or created or ip_source or ip_method or ip_reference):
is_add_ip = True
if is_add_ip == True:
data = {'domain': domain_name,
'campaign': campaign,
'confidence': confidence,
'domain_source': domain_source,
'domain_method': domain_method,
'domain_reference': domain_reference,
'add_ip': is_add_ip,
'ip': ip,
'ip_type': ip_type,
'created': created,
'same_source': is_same_source,
'ip_source': ip_source,
'ip_method': ip_method,
'ip_reference': ip_reference,
'add_indicators': is_add_indicators,
'bucket_list': bucket_list,
'ticket': ticket}
bound_domain_form = cache.get("domain_ip_form")
if bound_domain_form == None:
bound_domain_form = AddDomainForm(request.user, data)
cache['domain_ip_form'] = bound_domain_form
else:
bound_domain_form.data = data
else:
data = {'domain': domain_name,
'campaign': campaign,
'confidence': confidence,
'domain_source': domain_source,
'domain_method': domain_method,
'domain_reference': domain_reference,
'add_ip': is_add_ip,
'bucket_list': bucket_list,
'ticket': ticket}
bound_domain_form = cache.get("domain_form")
if bound_domain_form == None:
bound_domain_form = AddDomainForm(request.user, data)
cache['domain_form'] = bound_domain_form
else:
bound_domain_form.data = data
if bound_domain_form != None:
bound_domain_form.full_clean()
return bound_domain_form
def parse_bulk_upload(request,
parse_row_function,
add_new_function,
formdict,
cache={}):
"""Bulk adds data by parsing an input 2D array serially adding each row.
Args:
request: The Django context which contains information about the
session and the 2D data array for the handsontable bulk add request
parse_row_function: A callback function that will handle parsing
of each row's fields and values and then return the Django
form representation.
add_new_function: A callback function that will handle bulk adding
a single row.
formdict: The dictionary representation of the form
Returns:
Returns an array of dictionaries from the input form. Includes the
following keys in the dictionary:
failedRows: An array with a dictionary of key/value pairs with
information about which rows and why the failure occurred.
The following are the dictionary keys:
row: The row of the failure
col: The column of the failure. If the value is -1
then that means the column where the error occurred
is not determined.
label: The label of the column, if available
message: The reason for the failure
messages: Helpful messages about the bulk add operations
success: A boolean indicating if the bulk add operations were successful
successfulRows: An array with a dictionary of key/value pairs with
information about which rows were successfully bulk added.
The following are the dictionary keys:
row: The row that was successful
secondary: An array of information that can be passed from the
add_new_function() function call back to the caller of this
function. There are no restrictions on the data that can be
passed back. Usually this is used for post processing
analysis of the entire bulk add operation as a whole.
"""
failedRows = []
successfulRows = []
messages = []
secondaryData = []
isFailureDetected = False
is_validate_only = convert_string_to_bool(request.POST['isValidateOnly'])
offset = int(request.POST.get('offset', 0))
if cache.get("cleaned_rows_data"):
cleanedRowsData = cache.get("cleaned_rows_data")
else:
cleanedRowsData = convert_handsontable_to_rows(request)
rowCounter = offset
processedRows = 0
for rowData in cleanedRowsData:
rowCounter = rowCounter + 1
# Make sure that the rowData has content, otherwise the client side
# might have done some filtering to ignore rows that have no data.
if (rowData == None):
continue
try:
bound_form = parse_row_function(request, rowData, cache)
# If bound form passes validation checks then continue adding the item
if bound_form.is_valid():
data = bound_form.cleaned_data
errors = []
retVal = {}
message = ""
(result, errors,
retVal) = add_new_function(data, rowData, request, errors,
is_validate_only, cache)
processedRows += 1
#if retVal.get('message'):
# messages.append("At row " + str(rowCounter) + ": " + retVal.get('message'))
if retVal.get('secondary'):
secondaryData.append(retVal.get('secondary'))
# Check to make sure there were no errors
for error in errors:
message += error + '; '
else:
message = message[0:-2] # Remove '; ' from last
if errors:
# If there was an error then use a "col" value of -1, this is needed because
# there is no easy way to indicate the specific column that caused the error.
# The "col": -1 will just indicate to the client side to highlight the entire row.
failedRows.append({
'row':
rowCounter,
'col':
-1,
'message':
"At row " + str(rowCounter) + ": " + message
})
else:
status = retVal.get(form_consts.Status.STATUS_FIELD,
form_consts.Status.SUCCESS)
data = {
'row': rowCounter,
's': status
}
if retVal.get('warning'):
data['message'] = "At row " + str(
rowCounter) + ": " + retVal.get('warning')
elif retVal.get('message'):
data['message'] = "At row " + str(
rowCounter) + ": " + retVal.get('message')
successfulRows.append(data)
# ... otherwise validation of the bound form failed and we need to
# populate the response with error information
else:
processedRows += 1
for name, errorMessages in bound_form.errors.items():
entry = get_field_from_label(name, formdict)
if entry == None:
continue
for message in errorMessages:
failedRows.append({
'row':
rowCounter,
'col':
entry['position'],
'label':
entry['label'],
'message':
"At (" + str(rowCounter) + ", " + entry['label'] +
"): " + message
})
except Exception, e:
import traceback
traceback.print_exc()
