def process_bulk_add_domain(request, formdict): """ Performs the bulk add of domains by parsing the request data. Batches some data into a cache object for performance by reducing large amounts of single database queries. :param request: Django request. :type request: :class:`django.http.HttpRequest` :param formdict: The form representing the bulk uploaded data. :type formdict: dict :returns: :class:`django.http.HttpResponse` """ domain_names = [] ip_addresses = [] cached_domain_results = {} cached_ip_results = {} cleanedRowsData = convert_handsontable_to_rows(request) for rowData in cleanedRowsData: if rowData != None: if rowData.get(form_consts.Domain.DOMAIN_NAME) != None: domain = rowData.get( form_consts.Domain.DOMAIN_NAME).strip().lower() (root_domain, full_domain, error) = get_valid_root_domain(domain) domain_names.append(full_domain) if domain != root_domain: domain_names.append(root_domain) if rowData.get(form_consts.Domain.IP_ADDRESS) != None: ip_addr = rowData.get(form_consts.Domain.IP_ADDRESS) ip_type = rowData.get(form_consts.Domain.IP_TYPE) (ip_addr, error) = validate_and_normalize_ip(ip_addr, ip_type) ip_addresses.append(ip_addr) domain_results = Domain.objects(domain__in=domain_names) ip_results = IP.objects(ip__in=ip_addresses) for domain_result in domain_results: cached_domain_results[domain_result.domain] = domain_result for ip_result in ip_results: cached_ip_results[ip_result.ip] = ip_result cache = { form_consts.Domain.CACHED_RESULTS: cached_domain_results, form_consts.IP.CACHED_RESULTS: cached_ip_results, 'cleaned_rows_data': cleanedRowsData } response = parse_bulk_upload(request, parse_row_to_bound_domain_form, add_new_domain_via_bulk, formdict, cache) return response
def process_bulk_add_domain(request, formdict): """ Performs the bulk add of domains by parsing the request data. Batches some data into a cache object for performance by reducing large amounts of single database queries. :param request: Django request. :type request: :class:`django.http.HttpRequest` :param formdict: The form representing the bulk uploaded data. :type formdict: dict :returns: :class:`django.http.HttpResponse` """ domain_names = [] ip_addresses = [] cached_domain_results = {} cached_ip_results = {} cleanedRowsData = convert_handsontable_to_rows(request) for rowData in cleanedRowsData: if rowData != None: if rowData.get(form_consts.Domain.DOMAIN_NAME) != None: domain = rowData.get(form_consts.Domain.DOMAIN_NAME).strip().lower() (root_domain, full_domain, error) = get_valid_root_domain(domain) domain_names.append(full_domain) if domain != root_domain: domain_names.append(root_domain) if rowData.get(form_consts.Domain.IP_ADDRESS) != None: ip_addr = rowData.get(form_consts.Domain.IP_ADDRESS) ip_type = rowData.get(form_consts.Domain.IP_TYPE) (ip_addr, error) = validate_and_normalize_ip(ip_addr, ip_type) ip_addresses.append(ip_addr) domain_results = Domain.objects(domain__in=domain_names) ip_results = IP.objects(ip__in=ip_addresses) for domain_result in domain_results: cached_domain_results[domain_result.domain] = domain_result for ip_result in ip_results: cached_ip_results[ip_result.ip] = ip_result cache = { form_consts.Domain.CACHED_RESULTS: cached_domain_results, form_consts.IP.CACHED_RESULTS: cached_ip_results, "cleaned_rows_data": cleanedRowsData, } response = parse_bulk_upload(request, parse_row_to_bound_domain_form, add_new_domain_via_bulk, formdict, cache) return response
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}): """ Add a new domain to CRITs. :param data: The data about the domain. :type data: dict :param request: The Django request. :type request: :class:`django.http.HttpRequest` :param errors: A list of current errors to append to. :type errors: list :param rowData: Any objects that need to be added to the domain. :type rowData: dict :param is_validate_only: Only validate the data and return any errors. :type is_validate_only: boolean :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: tuple (<result>, <errors>, <retVal>) """ result = False retVal = {} domain = data['domain'] add_ip = data.get('add_ip') ip = data.get('ip') ip_type = data.get('ip_type') if add_ip: error = validate_and_normalize_ip(ip, ip_type)[1] if error: errors.append(error) if is_validate_only: error = get_valid_root_domain(domain)[2] if error: errors.append(error) # check for duplicate domains fqdn_domain = retrieve_domain(domain, cache) if fqdn_domain: if isinstance(fqdn_domain, Domain): resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) message = ('Warning: Domain already exists: ' '<a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message else: result_cache = cache.get(form_consts.Domain.CACHED_RESULTS) result_cache[domain.lower()] = True elif not errors: username = request.user.username reference = data.get('domain_reference') source_name = data.get('domain_source') method = data.get('domain_method') source = [ create_embedded_source(source_name, reference=reference, method=method, analyst=username) ] bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if data.get('campaign') and data.get('confidence'): campaign = [ EmbeddedCampaign(name=data.get('campaign'), confidence=data.get('confidence'), analyst=username) ] else: campaign = [] retVal = upsert_domain(domain, source, username, campaign, bucket_list=bucket_list, ticket=ticket, cache=cache) if not retVal['success']: errors.append(retVal.get('message')) retVal['message'] = "" else: new_domain = retVal['object'] ip_result = {} if add_ip: if data.get('same_source'): ip_source = source_name ip_method = method ip_reference = reference else: ip_source = data.get('ip_source') ip_method = data.get('ip_method') ip_reference = data.get('ip_reference') from crits.ips.handlers import ip_add_update ip_result = ip_add_update(ip, ip_type, ip_source, ip_method, ip_reference, campaign=campaign, analyst=username, bucket_list=bucket_list, ticket=ticket, cache=cache) if not ip_result['success']: errors.append(ip_result['message']) else: #add a relationship with the new IP address new_ip = ip_result['object'] if new_domain and new_ip: new_domain.add_relationship(rel_item=new_ip, rel_type='Resolved_To', analyst=username, get_rels=False) new_domain.save(username=username) new_ip.save(username=username) #set the URL for viewing the new data resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) if retVal['is_domain_new'] == True: retVal['message'] = ( 'Success! Click here to view the new domain: ' '<a href="%s">%s</a>' % (resp_url, domain)) else: message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal[form_consts.Status. STATUS_FIELD] = form_consts.Status.DUPLICATE retVal['warning'] = message #add indicators if data.get('add_indicators'): from crits.indicators.handlers import create_indicator_from_tlo # If we have an IP object, add an indicator for that. if ip_result.get('success'): ip = ip_result['object'] result = create_indicator_from_tlo('IP', ip, username, ip_source, add_domain=False) ip_ind = result.get('indicator') if not result['success']: errors.append(result['message']) # Add an indicator for the domain. result = create_indicator_from_tlo('Domain', new_domain, username, source_name, add_domain=False) if not result['success']: errors.append(result['message']) elif ip_result.get('success') and ip_ind: forge_relationship(left_class=result['indicator'], right_class=ip_ind, rel_type='Resolved_To', analyst=username) result = True # This block validates, and may also add, objects to the Domain if retVal.get('success') or is_validate_only == True: if rowData: objectsData = rowData.get(form_consts.Common.OBJECTS_DATA) # add new objects if they exist if objectsData: objectsData = json.loads(objectsData) current_domain = retrieve_domain(domain, cache) for object_row_counter, objectData in enumerate( objectsData, 1): if current_domain != None: # if the domain exists then try to add objects to it if isinstance(current_domain, Domain) == True: objectDict = object_array_to_dict( objectData, "Domain", current_domain.id) else: objectDict = object_array_to_dict( objectData, "Domain", "") current_domain = None else: objectDict = object_array_to_dict( objectData, "Domain", "") (obj_result, errors, obj_retVal) = validate_and_add_new_handler_object( None, objectDict, request, errors, object_row_counter, is_validate_only=is_validate_only, cache=cache, obj=current_domain) if not obj_result: retVal['success'] = False return result, errors, retVal
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}): """ Add a new domain to CRITs. :param data: The data about the domain. :type data: dict :param request: The Django request. :type request: :class:`django.http.HttpRequest` :param errors: A list of current errors to append to. :type errors: list :param rowData: Any objects that need to be added to the domain. :type rowData: dict :param is_validate_only: Only validate the data and return any errors. :type is_validate_only: boolean :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: tuple (<result>, <errors>, <retVal>) """ result = False retVal = {} domain = data['domain'] add_ip = data.get('add_ip') ip = data.get('ip') ip_type = data.get('ip_type') if add_ip: error = validate_and_normalize_ip(ip, ip_type)[1] if error: errors.append(error) if is_validate_only: error = get_valid_root_domain(domain)[2] if error: errors.append(error) # check for duplicate domains fqdn_domain = retrieve_domain(domain, cache) if fqdn_domain: if isinstance(fqdn_domain, Domain): resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) message = ('Warning: Domain already exists: ' '<a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message else: result_cache = cache.get(form_consts.Domain.CACHED_RESULTS); result_cache[domain.lower()] = True elif not errors: username = request.user.username reference = data.get('domain_reference') source_name = data.get('domain_source') method = data.get('domain_method') source = [create_embedded_source(source_name, reference=reference, method=method, analyst=username)] bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if data.get('campaign') and data.get('confidence'): campaign = [EmbeddedCampaign(name=data.get('campaign'), confidence=data.get('confidence'), analyst=username)] else: campaign = [] retVal = upsert_domain(domain, source, username, campaign, bucket_list=bucket_list, ticket=ticket, cache=cache) if not retVal['success']: errors.append(retVal.get('message')) retVal['message'] = "" else: new_domain = retVal['object'] ip_result = {} if add_ip: if data.get('same_source'): ip_source = source_name ip_method = method ip_reference = reference else: ip_source = data.get('ip_source') ip_method = data.get('ip_method') ip_reference = data.get('ip_reference') from crits.ips.handlers import ip_add_update ip_result = ip_add_update(ip, ip_type, ip_source, ip_method, ip_reference, campaign=campaign, analyst=username, bucket_list=bucket_list, ticket=ticket, cache=cache) if not ip_result['success']: errors.append(ip_result['message']) else: #add a relationship with the new IP address new_ip = ip_result['object'] if new_domain and new_ip: new_domain.add_relationship(new_ip, 'Resolved_To', analyst=username, get_rels=False) new_domain.save(username=username) #set the URL for viewing the new data resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) if retVal['is_domain_new'] == True: retVal['message'] = ('Success! Click here to view the new domain: ' '<a href="%s">%s</a>' % (resp_url, domain)) else: message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE retVal['warning'] = message #add indicators if data.get('add_indicators'): from crits.indicators.handlers import create_indicator_from_tlo # If we have an IP object, add an indicator for that. if ip_result.get('success'): ip = ip_result['object'] result = create_indicator_from_tlo('IP', ip, username, ip_source, add_domain=False) ip_ind = result.get('indicator') if not result['success']: errors.append(result['message']) # Add an indicator for the domain. result = create_indicator_from_tlo('Domain', new_domain, username, source_name, add_domain=False) if not result['success']: errors.append(result['message']) elif ip_result.get('success') and ip_ind: forge_relationship(left_class=result['indicator'], right_class=ip_ind, rel_type='Resolved_To', analyst=username) result = True # This block validates, and may also add, objects to the Domain if retVal.get('success') or is_validate_only == True: if rowData: objectsData = rowData.get(form_consts.Common.OBJECTS_DATA) # add new objects if they exist if objectsData: objectsData = json.loads(objectsData) current_domain = retrieve_domain(domain, cache) for object_row_counter, objectData in enumerate(objectsData, 1): if current_domain != None: # if the domain exists then try to add objects to it if isinstance(current_domain, Domain) == True: objectDict = object_array_to_dict(objectData, "Domain", current_domain.id) else: objectDict = object_array_to_dict(objectData, "Domain", "") current_domain = None; else: objectDict = object_array_to_dict(objectData, "Domain", "") (obj_result, errors, obj_retVal) = validate_and_add_new_handler_object( None, objectDict, request, errors, object_row_counter, is_validate_only=is_validate_only, cache=cache, obj=current_domain) if not obj_result: retVal['success'] = False return result, errors, retVal
def validate_indicator_value(value, ind_type): """ Check that a given value is valid for a particular Indicator type. :param value: The value to be validated :type value: str :param ind_type: The indicator type to validate against :type ind_type: str :returns: tuple: (Valid value, Error message) """ value = value.strip() domain = "" # URL if ind_type == IndicatorTypes.URI: if "://" not in value.split(".")[0]: return ("", "URI must contain protocol " "prefix (e.g. http://, https://, ftp://) ") domain_or_ip = urlparse.urlparse(value).hostname try: validate_ipv46_address(domain_or_ip) return (value, "") except DjangoValidationError: domain = domain_or_ip # Email address if ind_type in ( IndicatorTypes.EMAIL_ADDRESS, IndicatorTypes.EMAIL_FROM, IndicatorTypes.EMAIL_REPLY_TO, IndicatorTypes.EMAIL_SENDER, ): if "@" not in value: return ("", "Email address must contain an '@'") domain_or_ip = value.split("@")[-1] if domain_or_ip[0] == "[" and domain_or_ip[-1] == "]": try: validate_ipv46_address(domain_or_ip[1:-1]) return (value, "") except DjangoValidationError: return ("", "Email address does not contain a valid IP") else: domain = domain_or_ip # IPs if ind_type in IPTypes.values(): (ip_address, error) = validate_and_normalize_ip(value, ind_type) if error: return ("", error) else: return (ip_address, "") # Domains if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI) or domain: (root, domain, error) = get_valid_root_domain(domain or value) if error: return ("", error) else: return (value, "") return (value, "")