def obj_create(self, bundle, **kwargs): """ Handles creating IPs through the API. :param bundle: Bundle containing the information to create the IP. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ analyst = bundle.request.user.username data = bundle.data ip = data.get('ip', None) name = data.get('source', None) reference = data.get('reference', None) method = data.get('method', None) campaign = data.get('campaign', None) description = data.get('description', None) confidence = data.get('confidence', None) ip_type = data.get('ip_type', None) add_indicator = data.get('add_indicator', False) indicator_reference = data.get('indicator_reference', None) bucket_list = data.get('bucket_list', None) ticket = data.get('ticket', None) content = {'return_code': 1, 'type': 'IP'} if not ip or not name or not ip_type: content['message'] = "Must provide an IP, IP Type, and Source." self.crits_response(content) result = ip_add_update(ip, ip_type, source=name, source_method=method, source_reference=reference, campaign=campaign, confidence=confidence, description=description, analyst=analyst, bucket_list=bucket_list, ticket=ticket, is_add_indicator=add_indicator, indicator_reference=indicator_reference) if result.get('message'): content['message'] = result.get('message') if result.get('object'): content['id'] = str(result.get('object').id) if content.get('id'): url = reverse('api_dispatch_detail', kwargs={ 'resource_name': 'ips', 'api_name': 'v1', 'pk': content.get('id') }) content['url'] = url if result['success']: content['return_code'] = 0 self.crits_response(content)
def obj_create(self, bundle, **kwargs): """ Handles creating IPs through the API. :param bundle: Bundle containing the information to create the IP. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ analyst = bundle.request.user.username data = bundle.data ip = data.get('ip', None) name = data.get('source', None) reference = data.get('reference', None) method = data.get('method', None) campaign = data.get('campaign', None) description = data.get('description', None) confidence = data.get('confidence', None) ip_type = data.get('ip_type', None) add_indicator = data.get('add_indicator', False) indicator_reference = data.get('indicator_reference', None) bucket_list = data.get('bucket_list', None) ticket = data.get('ticket', None) content = {'return_code': 1, 'type': 'IP'} if not ip or not name or not ip_type: content['message'] = "Must provide an IP, IP Type, and Source." self.crits_response(content) result = ip_add_update(ip, ip_type, source=name, source_method=method, source_reference=reference, campaign=campaign, confidence=confidence, description=description, analyst=analyst, bucket_list=bucket_list, ticket=ticket, is_add_indicator=add_indicator, indicator_reference=indicator_reference) if result.get('message'): content['message'] = result.get('message') if result.get('object'): content['id'] = str(result.get('object').id) if content.get('id'): url = reverse('api_dispatch_detail', kwargs={'resource_name': 'ips', 'api_name': 'v1', 'pk': content.get('id')}) content['url'] = url if result['success']: content['return_code'] = 0 self.crits_response(content)
def obj_create(self, bundle, **kwargs): """ Handles creating IPs through the API. :param bundle: Bundle containing the information to create the IP. :type bundle: Tastypie Bundle object. :returns: Bundle object. :raises BadRequest: If creation fails. """ analyst = bundle.request.user.username data = bundle.data ip = data.get('ip', None) name = data.get('source', None) reference = data.get('reference', None) method = data.get('method', None) campaign = data.get('campaign', None) confidence = data.get('confidence', None) ip_type = data.get('ip_type', None) add_indicator = data.get('add_indicator', False) indicator_reference = data.get('indicator_reference', None) bucket_list = data.get('bucket_list', None) ticket = data.get('ticket', None) if not ip or not name or not ip_type: raise BadRequest("Must provide an IP, IP Type, and Source.") result = ip_add_update(ip, ip_type, source=name, source_method=method, source_reference=reference, campaign=campaign, confidence=confidence, analyst=analyst, bucket_list=bucket_list, ticket=ticket, is_add_indicator=add_indicator, indicator_reference=indicator_reference) if not result['success']: raise BadRequest(result['message']) else: return bundle
def process_ips(self, incident_report): ips = incident_report.findall("./contacted/ips/ip") for ip in ips: ret = ip_add_update(ip.text, self._ip_type(ip.text), source=get_user_organization(self.current_task.user), source_method=self.name, source_tlp=self.obj.tlp, user=self.current_task.user, related_id=str(self.obj.id), related_type=self.obj._meta['crits_type'], relationship_type=RelationshipTypes.CONNECTED_TO) if ret['success']: malicious = ip.get('malicious', 'unknown') self._add_result("IPs", ip.text, {'malicious': malicious}) else: self._warning(ret["message"]) self._notify()
def obj_create(self, bundle, **kwargs): """ Handles creating IPs through the API. :param bundle: Bundle containing the information to create the IP. :type bundle: Tastypie Bundle object. :returns: Bundle object. :raises BadRequest: If creation fails. """ analyst = bundle.request.user.username data = bundle.data ip = data['ip'] name = data['source'] reference = data['reference'] method = data['method'] campaign = data['campaign'] confidence = data['confidence'] ip_type = data['ip_type'] add_indicator = data.get('add_indicator', False) indicator_reference = data.get('indicator_reference') bucket_list = data.get('bucket_list', None) ticket = data.get('ticket', None) result = ip_add_update(ip, ip_type, source=name, source_method=method, source_reference=reference, campaign=campaign, confidence=confidence, analyst=analyst, bucket_list=bucket_list, ticket=ticket, is_add_indicator=add_indicator, indicator_reference=indicator_reference) if 'message' in result: raise BadRequest(result['message']) else: return bundle
def parse_observables(self, observables): """ Parse list of observables in STIX doc. :param observables: List of STIX observables. :type observables: List of STIX observables. """ analyst = self.source_instance.analyst for obs in observables: # for each STIX observable if obs.observable_composition: object_list = obs.observable_composition.observables else: object_list = [obs] for obs_comp in object_list: if not obs_comp.object_ or not obs_comp.object_.properties: self.failed.append( ("No valid object_properties was found!", type(obs_comp).__name__, obs_comp.id_)) # note for display in UI continue try: # try to create CRITs object from observable item = obs_comp.object_.properties if isinstance(item, Address): if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask', 'ipv6-subnet'): imp_type = "IP" for value in item.address_value.values: ip = str(value).strip() iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(ip, iptype, [self.source], analyst=analyst, id=self.package.id_) self.parse_res(imp_type, obs, res) if isinstance(item, DomainName): imp_type = "Domain" for value in item.value.values: res = upsert_domain(str(value), [self.source], username=analyst, id=self.package.id_) self.parse_res(imp_type, obs, res) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" rawdata = item.data.decode('utf-8') description = "None" # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ res = handle_raw_data_file( rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, obs, res) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if isinstance(obj.properties, Artifact): data = obj.properties.data res = handle_cert_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_NETWORK): data = obj.properties.data res = handle_pcap_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File): imp_type = "Sample" filename = str(item.file_name) md5 = item.md5 data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_FILE): data = obj.properties.data res = handle_file(filename, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, id=self.package.id_) self.parse_res(imp_type, obs, res) if item.extracted_features: self.parse_filenames(item.extracted_features, res['object'].id) elif isinstance(item, EmailMessage): imp_type = "Email" data = {} data['source'] = self.source.name data['source_method'] = self.source_instance.method data[ 'source_reference'] = self.source_instance.reference data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['message_id'] = str(item.header.message_id) data['subject'] = str(item.header.subject) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str( item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) data['date'] = item.header.date.value if item.header.to: data['to'] = [str(r) for r in item.header.to] if item.header.cc: data['cc'] = [str(r) for r in item.header.cc] res = handle_email_fields(data, analyst, "STIX", id=self.package.id_) # Should check for attachments and add them here. self.parse_res(imp_type, obs, res) if res.get('status') and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] self.relationships.append( (obs.id_, "Contains", rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" obj = make_crits_object(item) if obj.object_type == 'Address': # This was already caught above continue else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind( value.strip(), self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True) self.parse_res(imp_type, obs, res) except Exception, e: # probably caused by cybox object we don't handle self.failed.append( (e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}): """ Add a new domain to CRITs. :param data: The data about the domain. :type data: dict :param request: The Django request. :type request: :class:`django.http.HttpRequest` :param errors: A list of current errors to append to. :type errors: list :param rowData: Any objects that need to be added to the domain. :type rowData: dict :param is_validate_only: Only validate the data and return any errors. :type is_validate_only: boolean :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: tuple (<result>, <errors>, <retVal>) """ result = False retVal = {} domain = data['domain'] add_ip = data.get('add_ip') ip = data.get('ip') ip_type = data.get('ip_type') if add_ip: error = validate_and_normalize_ip(ip, ip_type)[1] if error: errors.append(error) if is_validate_only: error = get_valid_root_domain(domain)[2] if error: errors.append(error) # check for duplicate domains fqdn_domain = retrieve_domain(domain, cache) if fqdn_domain: if isinstance(fqdn_domain, Domain): resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) message = ('Warning: Domain already exists: ' '<a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message else: result_cache = cache.get(form_consts.Domain.CACHED_RESULTS) result_cache[domain.lower()] = True elif not errors: username = request.user.username reference = data.get('domain_reference') source_name = data.get('domain_source') method = data.get('domain_method') source = [ create_embedded_source(source_name, reference=reference, method=method, analyst=username) ] bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if data.get('campaign') and data.get('confidence'): campaign = [ EmbeddedCampaign(name=data.get('campaign'), confidence=data.get('confidence'), analyst=username) ] else: campaign = [] retVal = upsert_domain(domain, source, username, campaign, bucket_list=bucket_list, ticket=ticket, cache=cache) if not retVal['success']: errors.append(retVal.get('message')) retVal['message'] = "" else: new_domain = retVal['object'] ip_result = {} if add_ip: if data.get('same_source'): ip_source = source_name ip_method = method ip_reference = reference else: ip_source = data.get('ip_source') ip_method = data.get('ip_method') ip_reference = data.get('ip_reference') from crits.ips.handlers import ip_add_update ip_result = ip_add_update(ip, ip_type, ip_source, ip_method, ip_reference, campaign=campaign, analyst=username, bucket_list=bucket_list, ticket=ticket, cache=cache) if not ip_result['success']: errors.append(ip_result['message']) else: #add a relationship with the new IP address new_ip = ip_result['object'] if new_domain and new_ip: new_domain.add_relationship(rel_item=new_ip, rel_type='Resolved_To', analyst=username, get_rels=False) new_domain.save(username=username) new_ip.save(username=username) #set the URL for viewing the new data resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) if retVal['is_domain_new'] == True: retVal['message'] = ( 'Success! Click here to view the new domain: ' '<a href="%s">%s</a>' % (resp_url, domain)) else: message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal[form_consts.Status. STATUS_FIELD] = form_consts.Status.DUPLICATE retVal['warning'] = message #add indicators if data.get('add_indicators'): from crits.indicators.handlers import create_indicator_from_tlo # If we have an IP object, add an indicator for that. if ip_result.get('success'): ip = ip_result['object'] result = create_indicator_from_tlo('IP', ip, username, ip_source, add_domain=False) ip_ind = result.get('indicator') if not result['success']: errors.append(result['message']) # Add an indicator for the domain. result = create_indicator_from_tlo('Domain', new_domain, username, source_name, add_domain=False) if not result['success']: errors.append(result['message']) elif ip_result.get('success') and ip_ind: forge_relationship(left_class=result['indicator'], right_class=ip_ind, rel_type='Resolved_To', analyst=username) result = True # This block validates, and may also add, objects to the Domain if retVal.get('success') or is_validate_only == True: if rowData: objectsData = rowData.get(form_consts.Common.OBJECTS_DATA) # add new objects if they exist if objectsData: objectsData = json.loads(objectsData) current_domain = retrieve_domain(domain, cache) for object_row_counter, objectData in enumerate( objectsData, 1): if current_domain != None: # if the domain exists then try to add objects to it if isinstance(current_domain, Domain) == True: objectDict = object_array_to_dict( objectData, "Domain", current_domain.id) else: objectDict = object_array_to_dict( objectData, "Domain", "") current_domain = None else: objectDict = object_array_to_dict( objectData, "Domain", "") (obj_result, errors, obj_retVal) = validate_and_add_new_handler_object( None, objectDict, request, errors, object_row_counter, is_validate_only=is_validate_only, cache=cache, obj=current_domain) if not obj_result: retVal['success'] = False return result, errors, retVal
def add_update_ip(request, method): """ Add/update an IP address. Should be an AJAX POST. :param request: Django request. :type request: :class:`django.http.HttpRequest` :param method: If this is an "add" or an "update". :type method: str :returns: :class:`django.http.HttpResponse` """ if request.method == "POST" and request.is_ajax(): request.user._setup() data = request.POST form = AddIPForm(request.user, None, data) if form.is_valid(): cleaned_data = form.cleaned_data ip = cleaned_data['ip'] source_name = cleaned_data['source_name'] source_reference = cleaned_data['source_reference'] source_method = cleaned_data['source_method'] source_tlp = cleaned_data['source_tlp'] campaign = cleaned_data['campaign'] confidence = cleaned_data['confidence'] ip_type = cleaned_data['ip_type'] add_indicator = False if cleaned_data.get('add_indicator'): add_indicator = True indicator_reference = cleaned_data.get('indicator_reference') bucket_list = cleaned_data.get( form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME) related_id = cleaned_data['related_id'] related_type = cleaned_data['related_type'] relationship_type = cleaned_data['relationship_type'] result = ip_add_update(ip, ip_type, source=source_name, source_method=source_method, source_reference=source_reference, source_tlp=source_tlp, campaign=campaign, confidence=confidence, user=request.user, bucket_list=bucket_list, ticket=ticket, is_add_indicator=add_indicator, indicator_reference=indicator_reference, related_id=related_id, related_type=related_type, relationship_type=relationship_type) if 'message' in result: if not isinstance(result['message'], list): result['message'] = [result['message']] else: result['message'] = [] message = ( '<div>Success! Click here to view the new IP: <a ' 'href="%s">%s</a></div>' % (reverse('crits.ips.views.ip_detail', args=[ip]), ip)) result['message'].insert(0, message) return HttpResponse(json.dumps(result, default=json_handler), content_type="application/json") return HttpResponse(json.dumps({ 'success': False, 'form': form.as_table() }), content_type="application/json") return render_to_response("error.html", {'error': 'Expected AJAX/POST'}, RequestContext(request))
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}): """ Add a new domain to CRITs. :param data: The data about the domain. :type data: dict :param request: The Django request. :type request: :class:`django.http.HttpRequest` :param errors: A list of current errors to append to. :type errors: list :param rowData: Any objects that need to be added to the domain. :type rowData: dict :param is_validate_only: Only validate the data and return any errors. :type is_validate_only: boolean :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: tuple (<result>, <errors>, <retVal>) """ result = False retVal = {} domain = data['domain'] add_ip = data.get('add_ip') ip = data.get('ip') ip_type = data.get('ip_type') if add_ip: error = validate_and_normalize_ip(ip, ip_type)[1] if error: errors.append(error) if is_validate_only: error = get_valid_root_domain(domain)[2] if error: errors.append(error) # check for duplicate domains fqdn_domain = retrieve_domain(domain, cache) if fqdn_domain: if isinstance(fqdn_domain, Domain): resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) message = ('Warning: Domain already exists: ' '<a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message else: result_cache = cache.get(form_consts.Domain.CACHED_RESULTS); result_cache[domain.lower()] = True elif not errors: username = request.user.username reference = data.get('domain_reference') source_name = data.get('domain_source') method = data.get('domain_method') source = [create_embedded_source(source_name, reference=reference, method=method, analyst=username)] bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if data.get('campaign') and data.get('confidence'): campaign = [EmbeddedCampaign(name=data.get('campaign'), confidence=data.get('confidence'), analyst=username)] else: campaign = [] retVal = upsert_domain(domain, source, username, campaign, bucket_list=bucket_list, ticket=ticket, cache=cache) if not retVal['success']: errors.append(retVal.get('message')) retVal['message'] = "" else: new_domain = retVal['object'] ip_result = {} if add_ip: if data.get('same_source'): ip_source = source_name ip_method = method ip_reference = reference else: ip_source = data.get('ip_source') ip_method = data.get('ip_method') ip_reference = data.get('ip_reference') from crits.ips.handlers import ip_add_update ip_result = ip_add_update(ip, ip_type, ip_source, ip_method, ip_reference, campaign=campaign, analyst=username, bucket_list=bucket_list, ticket=ticket, cache=cache) if not ip_result['success']: errors.append(ip_result['message']) else: #add a relationship with the new IP address new_ip = ip_result['object'] if new_domain and new_ip: new_domain.add_relationship(new_ip, 'Resolved_To', analyst=username, get_rels=False) new_domain.save(username=username) #set the URL for viewing the new data resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) if retVal['is_domain_new'] == True: retVal['message'] = ('Success! Click here to view the new domain: ' '<a href="%s">%s</a>' % (resp_url, domain)) else: message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE retVal['warning'] = message #add indicators if data.get('add_indicators'): from crits.indicators.handlers import create_indicator_from_tlo # If we have an IP object, add an indicator for that. if ip_result.get('success'): ip = ip_result['object'] result = create_indicator_from_tlo('IP', ip, username, ip_source, add_domain=False) ip_ind = result.get('indicator') if not result['success']: errors.append(result['message']) # Add an indicator for the domain. result = create_indicator_from_tlo('Domain', new_domain, username, source_name, add_domain=False) if not result['success']: errors.append(result['message']) elif ip_result.get('success') and ip_ind: forge_relationship(left_class=result['indicator'], right_class=ip_ind, rel_type='Resolved_To', analyst=username) result = True # This block validates, and may also add, objects to the Domain if retVal.get('success') or is_validate_only == True: if rowData: objectsData = rowData.get(form_consts.Common.OBJECTS_DATA) # add new objects if they exist if objectsData: objectsData = json.loads(objectsData) current_domain = retrieve_domain(domain, cache) for object_row_counter, objectData in enumerate(objectsData, 1): if current_domain != None: # if the domain exists then try to add objects to it if isinstance(current_domain, Domain) == True: objectDict = object_array_to_dict(objectData, "Domain", current_domain.id) else: objectDict = object_array_to_dict(objectData, "Domain", "") current_domain = None; else: objectDict = object_array_to_dict(objectData, "Domain", "") (obj_result, errors, obj_retVal) = validate_and_add_new_handler_object( None, objectDict, request, errors, object_row_counter, is_validate_only=is_validate_only, cache=cache, obj=current_domain) if not obj_result: retVal['success'] = False return result, errors, retVal
def handle_indicator_insert(ind, source, reference='', analyst='', method='', add_domain=False, add_relationship=False, cache={}): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]: return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "} is_new_indicator = False dmain = None ip = None rank = { 'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4, } indicator = Indicator.objects(ind_type=ind['type'], value=ind['value']).first() if not indicator: indicator = Indicator() indicator.ind_type = ind['type'] indicator.value = ind['value'] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) is_new_indicator = True if 'campaign' in ind: if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0: confidence = ind.get('campaign_confidence', 'low') ind['campaign'] = EmbeddedCampaign(name=ind['campaign'], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now()) if isinstance(ind['campaign'], EmbeddedCampaign): indicator.add_campaign(ind['campaign']) elif isinstance(ind['campaign'], list): for campaign in ind['campaign']: if isinstance(campaign, EmbeddedCampaign): indicator.add_campaign(campaign) if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0): indicator.confidence.rating = ind['confidence'] indicator.confidence.analyst = analyst if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0): indicator.impact.rating = ind['impact'] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] if bucket_list: indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] if ticket: indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s, method=method, reference=reference) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source, method=method, reference=reference) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.value url_contains_ip = False if ind_type in ("URI - Domain Name", "URI - URL"): if ind_type == "URI - URL": domain_or_ip = urlparse.urlparse(ind_value).hostname elif ind_type == "URI - Domain Name": domain_or_ip = ind_value (sdomain, fqdn) = get_domain(domain_or_ip) if sdomain == "no_tld_found_error" and ind_type == "URI - URL": try: validate_ipv46_address(domain_or_ip) url_contains_ip = True except DjangoValidationError: pass if not url_contains_ip: success = None if add_domain: success = upsert_domain(sdomain, fqdn, indicator.source, '%s' % analyst, None, bucket_list=bucket_list, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: dmain = Domain.objects(domain=domain_or_ip).first() else: dmain = success['object'] if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip: if url_contains_ip: ind_value = domain_or_ip try: validate_ipv4_address(domain_or_ip) ind_type = 'Address - ipv4-addr' except DjangoValidationError: ind_type = 'Address - ipv6-addr' success = None if add_domain: success = ip_add_update(ind_value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: ip = IP.objects(ip=indicator.value).first() else: ip = success['object'] indicator.save(username=analyst) if dmain: dmain.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) if ip: ip.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) indicator.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(indicator, analyst) return {'success': True, 'objectid': str(indicator.id), 'is_new_indicator': is_new_indicator, 'object': indicator}
def parse_observables(self, observables): """ Parse list of observables in STIX doc. :param observables: List of STIX observables. :type observables: List of STIX observables. """ analyst = self.source_instance.analyst for obs in observables: # for each STIX observable if not obs.object_ or not obs.object_.properties: self.failed.append(("No valid object_properties was found!", type(obs).__name__, obs.id_)) # note for display in UI continue try: # try to create CRITs object from observable item = obs.object_.properties if isinstance(item, Address): if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: ip = str(value).strip() iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(ip, iptype, [self.source], analyst=analyst, is_add_indicator=True) self.parse_res(imp_type, obs, res) if isinstance(item, DomainName): imp_type = "Domain" for value in item.value.values: res = upsert_domain(str(value), [self.source], username=analyst) self.parse_res(imp_type, obs, res) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" rawdata = item.data.decode('utf-8') description = "None" # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ res = handle_raw_data_file(rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, obs, res) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if isinstance(obj.properties, Artifact): data = obj.properties.data res = handle_cert_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_NETWORK): data = obj.properties.data res = handle_pcap_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File): imp_type = "Sample" filename = str(item.file_name) md5 = item.md5 data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_FILE): data = obj.properties.data res = handle_file(filename, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False) self.parse_res(imp_type, obs, res) elif isinstance(item, EmailMessage): imp_type = "Email" data = {} data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['message_id'] = str(item.header.message_id) data['subject'] = str(item.header.subject) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str(item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) data['date'] = item.header.date.value if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] res = handle_email_fields(data, analyst, "STIX") # Should check for attachments and add them here. self.parse_res(imp_type, obs, res) if res.get('status') and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] self.relationships.append((obs.id_, "Contains", rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" obj = make_crits_object(item) if obj.object_type == 'Address': # This was already caught above continue else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind(value.strip(), self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True) self.parse_res(imp_type, obs, res) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def handle_indicator_insert( ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={} ): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ if ind["type"] not in IndicatorTypes.values(): return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]} if ind["threat_type"] not in IndicatorThreatTypes.values(): return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]} if ind["attack_type"] not in IndicatorAttackTypes.values(): return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]} (ind["value"], error) = validate_indicator_value(ind["value"], ind["type"]) if error: return {"success": False, "message": error} is_new_indicator = False dmain = None ip = None rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4} if ind.get("status", None) is None or len(ind.get("status", "")) < 1: ind["status"] = Status.NEW indicator = Indicator.objects( ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"] ).first() if not indicator: indicator = Indicator() indicator.ind_type = ind["type"] indicator.threat_type = ind["threat_type"] indicator.attack_type = ind["attack_type"] indicator.value = ind["value"] indicator.lower = ind["lower"] indicator.description = ind["description"] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) indicator.status = ind["status"] is_new_indicator = True else: if ind["status"] != Status.NEW: indicator.status = ind["status"] add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"]) if indicator.description is None: indicator.description = add_desc else: indicator.description += add_desc if "campaign" in ind: if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0: confidence = ind.get("campaign_confidence", "low") ind["campaign"] = EmbeddedCampaign( name=ind["campaign"], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now(), ) if isinstance(ind["campaign"], EmbeddedCampaign): indicator.add_campaign(ind["campaign"]) elif isinstance(ind["campaign"], list): for campaign in ind["campaign"]: if isinstance(campaign, EmbeddedCampaign): indicator.add_campaign(campaign) if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0): indicator.confidence.rating = ind["confidence"] indicator.confidence.analyst = analyst if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0): indicator.impact.rating = ind["impact"] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] if bucket_list: indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] if ticket: indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s, method=method, reference=reference) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source, method=method, reference=reference) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.lower url_contains_ip = False if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI): if ind_type == IndicatorTypes.URI: domain_or_ip = urlparse.urlparse(ind_value).hostname try: validate_ipv46_address(domain_or_ip) url_contains_ip = True except DjangoValidationError: pass else: domain_or_ip = ind_value if not url_contains_ip: success = None if add_domain: success = upsert_domain( domain_or_ip, indicator.source, username="******" % analyst, campaign=indicator.campaign, bucket_list=bucket_list, cache=cache, ) if not success["success"]: return {"success": False, "message": success["message"]} if not success or not "object" in success: dmain = Domain.objects(domain=domain_or_ip).first() else: dmain = success["object"] if ind_type in IPTypes.values() or url_contains_ip: if url_contains_ip: ind_value = domain_or_ip try: validate_ipv4_address(domain_or_ip) ind_type = IndicatorTypes.IPV4_ADDRESS except DjangoValidationError: ind_type = IndicatorTypes.IPV6_ADDRESS success = None if add_domain: success = ip_add_update( ind_value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache, ) if not success["success"]: return {"success": False, "message": success["message"]} if not success or not "object" in success: ip = IP.objects(ip=indicator.value).first() else: ip = success["object"] indicator.save(username=analyst) if dmain: dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) if ip: ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(indicator, analyst) return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
def handle_indicator_insert(ind, source, reference='', analyst='', method='', add_domain=False, add_relationship=False, cache={}): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ is_new_indicator = False rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4} indicator = Indicator.objects(ind_type=ind['type'], value=ind['value']).first() if not indicator: indicator = Indicator() indicator.ind_type = ind['type'] indicator.value = ind['value'] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) is_new_indicator = True ec = None if 'campaign' in ind: confidence = 'low' if 'campaign_confidence' in ind: confidence = ind['campaign_confidence'] ec = EmbeddedCampaign(name=ind['campaign'], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now()) if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get( indicator.confidence.rating, 0): indicator.confidence.rating = ind['confidence'] indicator.confidence.analyst = analyst if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get( indicator.impact.rating, 0): indicator.impact.rating = ind['impact'] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if ec: indicator.add_campaign(ec) indicator.save(username=analyst) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.value if ind_type in ("URI - Domain Name", "URI - URL"): if ind_type == "URI - URL": domain = ind_value.split("/")[2] elif ind_type == "URI - Domain Name": domain = ind_value #try: (sdomain, fqdn) = get_domain(domain) success = None if add_domain: success = upsert_domain(sdomain, fqdn, indicator.source, '%s' % analyst, None, bucket_list=bucket_list, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: dmain = Domain.objects(domain=domain).first() else: dmain = success['object'] if dmain: dmain.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) indicator.save(username=analyst) elif ind_type.startswith( "Address - ip") or ind_type == "Address - cidr": success = None if add_domain: success = ip_add_update(indicator.value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: ip = IP.objects(ip=indicator.value).first() else: ip = success['object'] if ip: ip.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) indicator.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(None, indicator, analyst) return { 'success': True, 'objectid': indicator.id, 'is_new_indicator': is_new_indicator, 'object': indicator }
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}): """ Add a new domain to CRITs. :param data: The data about the domain. :type data: dict :param request: The Django request. :type request: :class:`django.http.HttpRequest` :param errors: A list of current errors to append to. :type errors: list :param rowData: Any objects that need to be added to the domain. :type rowData: dict :param is_validate_only: Only validate the data and return any errors. :type is_validate_only: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: tuple """ username = request.user.username result = False retVal = {} reference = data.get('domain_reference') name = data.get('domain_source') method = data.get('domain_method') source = [create_embedded_source(name, reference=reference, method=method, analyst=username)] bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if data.get('campaign') and data.get('confidence'): campaign = [EmbeddedCampaign(name=data.get('campaign'), confidence=data.get('confidence'), analyst=username)] else: campaign = [] (sdomain, fqdn) = get_domain(data['domain']) if sdomain == "no_tld_found_error": errors.append(u"Error: Invalid domain: " + data['domain']) elif is_validate_only == False: retVal = upsert_domain(sdomain, fqdn, source, username, campaign, bucket_list=bucket_list, ticket=ticket, cache=cache) ip_result = None if retVal['success']: add_ip = data.get('add_ip') if add_ip: ip = data.get('ip') if data.get('same_source'): ip_source = data.get('domain_source') ip_method = data.get('domain_method') ip_reference = reference else: ip_source = data.get('ip_source') ip_method = data.get('ip_method') ip_reference = data.get('ip_reference') from crits.ips.handlers import ip_add_update ip_result = ip_add_update(ip, None, ip_source, ip_method, ip_reference, campaign=campaign, analyst=username, bucket_list=bucket_list, ticket=ticket, cache=cache) if not ip_result['success']: errors.append(ip_result['message']) else: #add a relationship with the new IP address new_domain = retVal['object'] new_ip = ip_result['object'] if new_domain and new_ip: new_domain.add_relationship(rel_item=new_ip, rel_type='Resolved_To', analyst=username, get_rels=False) new_domain.save(username=username) new_ip.save(username=username) #set the URL for viewing the new data resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn]) if retVal['is_domain_new'] == True: retVal['message'] = ('Success! Click here to view the new domain: ' '<a href="%s">%s</a>' % (resp_url, fqdn)) else: message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, fqdn)) retVal['message'] = message retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE retVal['warning'] = message #add indicators # add_indicators_for_domain handles adding relationships as well if data.get('add_indicators'): analyst = username from crits.indicators.handlers import add_indicators_for_domain if ip_result and ip_result['success']: obj = ip_result['object'] errors += add_indicators_for_domain(sdomain, fqdn, source, analyst, reference, obj, bucket_list=bucket_list, ticket=ticket)['errors'] else: errors += add_indicators_for_domain(sdomain, fqdn, source, analyst, reference, bucket_list=bucket_list, ticket=ticket)['errors'] result = True elif 'message' in retVal: #database error? (!c_dom) errors.append(retVal['message']) #u"Unknown error: unable to add domain") elif is_validate_only == True: domain = data['domain'] fqdn_domain = retrieve_domain(domain, cache); if fqdn_domain: if isinstance(fqdn_domain, Domain): resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn]) message = ('Warning: Domain already exists: ' '<a href="%s">%s</a>' % (resp_url, fqdn)) retVal['message'] = message; retVal['status'] = form_consts.Status.DUPLICATE; retVal['warning'] = message; else: result_cache = cache.get(form_consts.Domain.CACHED_RESULTS); result_cache[domain.lower()] = True; # This block tries to add objects to the item if retVal.get('success') or is_validate_only == True: if rowData: objectsData = rowData.get(form_consts.Common.OBJECTS_DATA) # add new objects if they exist if objectsData: objectsData = json.loads(objectsData) object_row_counter = 1 current_domain = retrieve_domain(fqdn, cache) for objectData in objectsData: if current_domain != None: # if the domain exists then try to add objects to it if isinstance(current_domain, Domain) == True: objectDict = object_array_to_dict(objectData, "Domain", current_domain.id) else: objectDict = object_array_to_dict(objectData, "Domain", "") current_domain = None; else: objectDict = object_array_to_dict(objectData, "Domain", "") (object_result, object_errors, object_retVal) = validate_and_add_new_handler_object( None, objectDict, request, errors, object_row_counter, is_validate_only=is_validate_only, cache=cache, obj=current_domain) if object_retVal.get('success') == False: retVal['success'] = False if object_retVal.get('message'): errors.append(object_retVal['message']) object_row_counter += 1 return result, errors, retVal
def add_update_ip(request, method): """ Add/update an IP address. Should be an AJAX POST. :param request: Django request. :type request: :class:`django.http.HttpRequest` :param method: If this is an "add" or an "update". :type method: str :returns: :class:`django.http.HttpResponse` """ if request.method == "POST" and request.is_ajax(): request.user._setup() data = request.POST form = AddIPForm(request.user, None, data) if form.is_valid(): cleaned_data = form.cleaned_data ip = cleaned_data['ip'] source_name = cleaned_data['source_name'] source_reference = cleaned_data['source_reference'] source_method = cleaned_data['source_method'] source_tlp= cleaned_data['source_tlp'] campaign = cleaned_data['campaign'] confidence = cleaned_data['confidence'] ip_type = cleaned_data['ip_type'] add_indicator = False if cleaned_data.get('add_indicator'): add_indicator = True indicator_reference = cleaned_data.get('indicator_reference') bucket_list = cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME) related_id = cleaned_data['related_id'] related_type = cleaned_data['related_type'] relationship_type = cleaned_data['relationship_type'] result = ip_add_update(ip, ip_type, source=source_name, source_method=source_method, source_reference=source_reference, source_tlp=source_tlp, campaign=campaign, confidence=confidence, user=request.user, bucket_list=bucket_list, ticket=ticket, is_add_indicator=add_indicator, indicator_reference=indicator_reference, related_id=related_id, related_type=related_type, relationship_type=relationship_type) if 'message' in result: if not isinstance(result['message'], list): result['message'] = [result['message']] else: result['message'] = [] message = ('<div>Success! Click here to view the new IP: <a ' 'href="%s">%s</a></div>' % (reverse('crits.ips.views.ip_detail', args=[ip]), ip)) result['message'].insert(0, message) return HttpResponse(json.dumps(result, default=json_handler), content_type="application/json") return HttpResponse(json.dumps({'success': False, 'form':form.as_table()}), content_type="application/json") return render_to_response("error.html", {'error': 'Expected AJAX/POST'}, RequestContext(request))
def parse_cybox_object(self, cbx_obj, description='', ind_id=None): """ Parse a CybOX object form a STIX doc. An object can contain multiple related_objects, which in turn can have their own related_objects, so this handles those recursively. :param cbx_obj: The CybOX object to parse. :type cbx_obj: A CybOX object. :param description: Parent-level (e.g. Observable) description. :type description: str :param ind_id: The ID of a parent STIX Indicator. :type ind_id: str """ # check for missing attributes if not cbx_obj or not cbx_obj.properties: if cbx_obj.idref: # just a reference, so nothing to parse return else: cbx_id = getattr(cbx_obj, 'id_', 'None') self.failed.append(("No valid object_properties was found!", "Observable (%s)" % cbx_id, cbx_id)) # note for display in UI return # Don't parse if already been parsed # This is for artifacts that are related to CybOX File Objects if cbx_obj.id_ in self.parsed: return try: # try to create CRITs object from Cybox Object analyst = self.source_instance.analyst item = cbx_obj.properties val = cbx_obj.id_ if isinstance(item, Address) and not ind_id: if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: val = str(value).strip() if self.preview: res = None else: iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(val, iptype, [self.source], analyst=analyst, is_add_indicator=True) else: res = { 'success': False, 'reason': 'No IP Type' } self.parse_res(imp_type, val, cbx_obj, res, ind_id) if (not ind_id and (isinstance(item, DomainName) or (isinstance(item, URI) and item.type_ == 'Domain Name'))): imp_type = "Domain" for val in item.value.values: if self.preview: res = None else: res = upsert_domain(str(val), [self.source], username=analyst) self.parse_res(imp_type, str(val), cbx_obj, res, ind_id) elif isinstance(item, HTTPSession): imp_type = "RawData" val = cbx_obj.id_ try: c_req = item.http_request_response[0].http_client_request hdr = c_req.http_request_header if hdr.raw_header: data = hdr.raw_header.value title = "HTTP Header from STIX: %s" % self.package.id_ method = self.source_instance.method ref = self.source_instance.reference if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="HTTP Header", tool_name="STIX", tool_version=None, method=method, reference=ref) else: imp_type = "Indicator" ind_type = "HTTP Request Header Fields - User-Agent" val = hdr.parsed_header.user_agent.value val = ','.join(val) if isinstance(val, list) else val if self.preview: res = None else: res = handle_indicator_ind( val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_relationship=True, description=description) except: msg = "Unsupported use of 'HTTPSession' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, WhoisEntry): # No sure where else to put this imp_type = "RawData" val = cbx_obj.id_ if item.remarks: data = item.remarks.value title = "WHOIS Entry from STIX: %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file( data, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="WHOIS", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) else: msg = "Unsupported use of 'WhoisEntry' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" val = cbx_obj.id_ rawdata = item.data.decode('utf-8') # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file( rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if isinstance(rel_obj.properties, Artifact): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_cert_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_NETWORK): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_pcap_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File): imp_type = "Sample" md5 = item.md5 if md5: md5 = md5.lower() val = str(item.file_name or md5) # add sha1/sha256/ssdeep once handle_file supports it size = item.size_in_bytes data = None if item.file_path: path = "File Path: " + str(item.file_path) description += "\n" + path for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_FILE): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) if not md5 and not data and val and val != "None": imp_type = "Indicator" if self.preview: res = None else: res = handle_indicator_ind( val, self.source, "Win File", IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) elif md5 or data: if self.preview: res = None else: res = handle_file(val, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, size=size, description=description) else: val = cbx_obj.id_ msg = "CybOX 'File' object has no MD5, data, or filename" res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, EmailMessage): imp_type = 'Email' id_list = [] data = {} val = cbx_obj.id_ get_attach = False data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['subject'] = str(item.header.subject) if item.header.date: data['date'] = item.header.date.value val = "Date: %s, Subject: %s" % (data.get( 'date', 'None'), data['subject']) data['message_id'] = str(item.header.message_id) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str( item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] if data.get('date'): # Email TLOs must have a date data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference if self.preview: res = None else: res = handle_email_fields(data, analyst, "STIX") self.parse_res(imp_type, val, cbx_obj, res, ind_id) if not self.preview and res.get('status'): id_list.append(cbx_obj.id_) # save ID for atchmnt rels get_attach = True else: # Can't be an Email TLO, so save fields for x, key in enumerate(data): if data[key] and data[key] != "None": if key in ('raw_header', 'raw_body'): if key == 'raw_header': title = "Email Header from STIX Email: %s" d_type = "Email Header" else: title = "Email Body from STIX Email: %s" d_type = "Email Body" imp_type = 'RawData' title = title % cbx_obj.id_ if self.preview: res = None else: res = handle_raw_data_file( data[key], self.source, analyst, description, title, d_type, "STIX", self.stix_version) self.parse_res(imp_type, title, cbx_obj, res, ind_id) elif key == 'to': imp_type = 'Target' for y, addr in enumerate(data[key]): tgt_dict = {'email_address': addr} if self.preview: res = None else: res = upsert_target(tgt_dict, analyst) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x, y) self.parse_res(imp_type, addr, tmp_obj, res, ind_id) self.ind2obj.setdefault( cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) else: imp_type = 'Indicator' if key in ('sender', 'reply_to', 'from_address'): ind_type = "Address - e-mail" elif 'ip' in key: ind_type = "Address - ipv4-addr" elif key == 'raw_body': ind_type = "Email Message" else: ind_type = "String" if self.preview: res = None else: res = handle_indicator_ind( data[key], self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x) self.parse_res(imp_type, data[key], tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) if not self.preview: # Setup relationships between all Email attributes for oid in id_list: for oid2 in id_list: if oid != oid2: self.relationships.append( (oid, RelationshipTypes.RELATED_TO, oid2, "High")) # Should check for attachments and add them here. if get_attach and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] for oid in id_list: self.relationships.append( (oid, RelationshipTypes.CONTAINS, rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" val = cbx_obj.id_ c_obj = make_crits_object(item) # Ignore what was already caught above if (ind_id or c_obj.object_type not in IPTypes.values()): ind_type = c_obj.object_type for val in [str(v).strip() for v in c_obj.value if v]: if ind_type: # handle domains mislabeled as URLs if c_obj.object_type == 'URI' and '/' not in val: ind_type = "Domain" if self.preview: res = None else: res = handle_indicator_ind( val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, "%s (%s)" % (imp_type, val), cbx_obj.id_)) # note for display in UI
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}): """ Add a new domain to CRITs. :param data: The data about the domain. :type data: dict :param request: The Django request. :type request: :class:`django.http.HttpRequest` :param errors: A list of current errors to append to. :type errors: list :param rowData: Any objects that need to be added to the domain. :type rowData: dict :param is_validate_only: Only validate the data and return any errors. :type is_validate_only: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: tuple """ username = request.user.username result = False retVal = {} reference = data.get('domain_reference') name = data.get('domain_source') method = data.get('domain_method') source = [ create_embedded_source(name, reference=reference, method=method, analyst=username) ] bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if data.get('campaign') and data.get('confidence'): campaign = [ EmbeddedCampaign(name=data.get('campaign'), confidence=data.get('confidence'), analyst=username) ] else: campaign = [] (sdomain, fqdn) = get_domain(data['domain']) if sdomain == "no_tld_found_error": errors.append(u"Error: Invalid domain: " + data['domain']) elif is_validate_only == False: retVal = upsert_domain(sdomain, fqdn, source, username, campaign, bucket_list=bucket_list, ticket=ticket, cache=cache) ip_result = None if retVal['success']: new_domain = retVal['object'] add_ip = data.get('add_ip') if add_ip: ip = data.get('ip') ip_type = data.get('ip_type') if data.get('same_source'): ip_source = data.get('domain_source') ip_method = data.get('domain_method') ip_reference = reference else: ip_source = data.get('ip_source') ip_method = data.get('ip_method') ip_reference = data.get('ip_reference') from crits.ips.handlers import ip_add_update ip_result = ip_add_update(ip, ip_type, ip_source, ip_method, ip_reference, campaign=campaign, analyst=username, bucket_list=bucket_list, ticket=ticket, cache=cache) if not ip_result['success']: errors.append(ip_result['message']) else: #add a relationship with the new IP address new_ip = ip_result['object'] if new_domain and new_ip: new_domain.add_relationship(rel_item=new_ip, rel_type='Resolved_To', analyst=username, get_rels=False) new_domain.save(username=username) new_ip.save(username=username) #set the URL for viewing the new data resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn]) if retVal['is_domain_new'] == True: retVal['message'] = ( 'Success! Click here to view the new domain: ' '<a href="%s">%s</a>' % (resp_url, fqdn)) else: message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, fqdn)) retVal['message'] = message retVal[form_consts.Status. STATUS_FIELD] = form_consts.Status.DUPLICATE retVal['warning'] = message #add indicators if data.get('add_indicators'): from crits.indicators.handlers import create_indicator_from_obj # If we have an IP object, add an indicator for that. if ip_result and ip_result['success']: obj = ip_result['object'] result = create_indicator_from_obj('Address - ipv4-addr', 'IP', obj.id, obj.ip, username) if result['success'] == False: errors.append(result['message']) # Add an indicator for the domain. result = create_indicator_from_obj('URI - Domain Name', 'Domain', new_domain.id, sdomain, username) if result['success'] == False: errors.append(result['message']) # If we have an FQDN (ie: it is not the same as sdomain) # then add that also. if fqdn != sdomain: result = create_indicator_from_obj('URI - Domain Name', 'Domain', new_domain.id, fqdn, username) if result['success'] == False: errors.append(result['message']) result = True elif 'message' in retVal: #database error? (!c_dom) errors.append( retVal['message']) #u"Unknown error: unable to add domain") elif is_validate_only == True: domain = data['domain'] fqdn_domain = retrieve_domain(domain, cache) if fqdn_domain: if isinstance(fqdn_domain, Domain): resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn]) message = ('Warning: Domain already exists: ' '<a href="%s">%s</a>' % (resp_url, fqdn)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message else: result_cache = cache.get(form_consts.Domain.CACHED_RESULTS) result_cache[domain.lower()] = True # This block tries to add objects to the item if retVal.get('success') or is_validate_only == True: if rowData: objectsData = rowData.get(form_consts.Common.OBJECTS_DATA) # add new objects if they exist if objectsData: objectsData = json.loads(objectsData) current_domain = retrieve_domain(fqdn, cache) for object_row_counter, objectData in enumerate( objectsData, 1): if current_domain != None: # if the domain exists then try to add objects to it if isinstance(current_domain, Domain) == True: objectDict = object_array_to_dict( objectData, "Domain", current_domain.id) else: objectDict = object_array_to_dict( objectData, "Domain", "") current_domain = None else: objectDict = object_array_to_dict( objectData, "Domain", "") (object_result, object_errors, object_retVal) = validate_and_add_new_handler_object( None, objectDict, request, errors, object_row_counter, is_validate_only=is_validate_only, cache=cache, obj=current_domain) if object_retVal.get('success') == False: retVal['success'] = False if object_retVal.get('message'): errors.append(object_retVal['message']) return result, errors, retVal
def parse_cybox_object(self, cbx_obj, description='', ind_id=None): """ Parse a CybOX object form a STIX doc. An object can contain multiple related_objects, which in turn can have their own related_objects, so this handles those recursively. :param cbx_obj: The CybOX object to parse. :type cbx_obj: A CybOX object. :param description: Parent-level (e.g. Observable) description. :type description: str :param ind_id: The ID of a parent STIX Indicator. :type ind_id: str """ # check for missing attributes if not cbx_obj or not cbx_obj.properties: if cbx_obj.idref: # just a reference, so nothing to parse return else: cbx_id = getattr(cbx_obj, 'id_', 'None') self.failed.append(("No valid object_properties was found!", "Observable (%s)" % cbx_id, cbx_id)) # note for display in UI return # Don't parse if already been parsed # This is for artifacts that are related to CybOX File Objects if cbx_obj.id_ in self.parsed: return try: # try to create CRITs object from Cybox Object analyst = self.source_instance.analyst item = cbx_obj.properties val = cbx_obj.id_ if isinstance(item, Address) and not ind_id: if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: val = str(value).strip() if self.preview: res = None else: iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(val, iptype, [self.source], analyst=analyst, is_add_indicator=True) else: res = {'success': False, 'reason': 'No IP Type'} self.parse_res(imp_type, val, cbx_obj, res, ind_id) if (not ind_id and (isinstance(item, DomainName) or (isinstance(item, URI) and item.type_ == 'Domain Name'))): imp_type = "Domain" for val in item.value.values: if self.preview: res = None else: res = upsert_domain(str(val), [self.source], username=analyst) self.parse_res(imp_type, str(val), cbx_obj, res, ind_id) elif isinstance(item, HTTPSession): imp_type = "RawData" val = cbx_obj.id_ try: c_req = item.http_request_response[0].http_client_request hdr = c_req.http_request_header if hdr.raw_header: data = hdr.raw_header.value title = "HTTP Header from STIX: %s" % self.package.id_ method = self.source_instance.method ref = self.source_instance.reference if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="HTTP Header", tool_name="STIX", tool_version=None, method=method, reference=ref) else: imp_type = "Indicator" ind_type = "HTTP Request Header Fields - User-Agent" val = hdr.parsed_header.user_agent.value val = ','.join(val) if isinstance(val, list) else val if self.preview: res = None else: res = handle_indicator_ind(val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_relationship=True, description=description) except: msg = "Unsupported use of 'HTTPSession' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, WhoisEntry): # No sure where else to put this imp_type = "RawData" val = cbx_obj.id_ if item.remarks: data = item.remarks.value title = "WHOIS Entry from STIX: %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="WHOIS", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) else: msg = "Unsupported use of 'WhoisEntry' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" val = cbx_obj.id_ rawdata = item.data.decode('utf-8') # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file(rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if isinstance(rel_obj.properties, Artifact): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_cert_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_NETWORK): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_pcap_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File): imp_type = "Sample" md5 = item.md5 if md5: md5 = md5.lower() val = str(item.file_name or md5) # add sha1/sha256/ssdeep once handle_file supports it size = item.size_in_bytes data = None if item.file_path: path = "File Path: " + str(item.file_path) description += "\n" + path for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_FILE): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) if not md5 and not data and val and val != "None": imp_type = "Indicator" if self.preview: res = None else: res = handle_indicator_ind(val, self.source, "Win File", IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) elif md5 or data: if self.preview: res = None else: res = handle_file(val, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, size=size, description=description) else: val = cbx_obj.id_ msg = "CybOX 'File' object has no MD5, data, or filename" res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, EmailMessage): imp_type = 'Email' id_list = [] data = {} val = cbx_obj.id_ get_attach = False data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['subject'] = str(item.header.subject) if item.header.date: data['date'] = item.header.date.value val = "Date: %s, Subject: %s" % (data.get('date', 'None'), data['subject']) data['message_id'] = str(item.header.message_id) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str(item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] if data.get('date'): # Email TLOs must have a date data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference if self.preview: res = None else: res = handle_email_fields(data, analyst, "STIX") self.parse_res(imp_type, val, cbx_obj, res, ind_id) if not self.preview and res.get('status'): id_list.append(cbx_obj.id_) # save ID for atchmnt rels get_attach = True else: # Can't be an Email TLO, so save fields for x, key in enumerate(data): if data[key] and data[key] != "None": if key in ('raw_header', 'raw_body'): if key == 'raw_header': title = "Email Header from STIX Email: %s" d_type = "Email Header" else: title = "Email Body from STIX Email: %s" d_type = "Email Body" imp_type = 'RawData' title = title % cbx_obj.id_ if self.preview: res = None else: res = handle_raw_data_file(data[key], self.source, analyst, description, title, d_type, "STIX", self.stix_version) self.parse_res(imp_type, title, cbx_obj, res, ind_id) elif key == 'to': imp_type = 'Target' for y, addr in enumerate(data[key]): tgt_dict = {'email_address': addr} if self.preview: res = None else: res = upsert_target(tgt_dict, analyst) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x, y) self.parse_res(imp_type, addr, tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) else: imp_type = 'Indicator' if key in ('sender', 'reply_to', 'from_address'): ind_type = "Address - e-mail" elif 'ip' in key: ind_type = "Address - ipv4-addr" elif key == 'raw_body': ind_type = "Email Message" else: ind_type = "String" if self.preview: res = None else: res = handle_indicator_ind(data[key], self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x) self.parse_res(imp_type, data[key], tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) if not self.preview: # Setup relationships between all Email attributes for oid in id_list: for oid2 in id_list: if oid != oid2: self.relationships.append((oid, RelationshipTypes.RELATED_TO, oid2, "High")) # Should check for attachments and add them here. if get_attach and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] for oid in id_list: self.relationships.append((oid, RelationshipTypes.CONTAINS, rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" val = cbx_obj.id_ c_obj = make_crits_object(item) # Ignore what was already caught above if (ind_id or c_obj.object_type not in IPTypes.values()): ind_type = c_obj.object_type for val in [str(v).strip() for v in c_obj.value if v]: if ind_type: # handle domains mislabeled as URLs if c_obj.object_type == 'URI' and '/' not in val: ind_type = "Domain" if self.preview: res = None else: res = handle_indicator_ind(val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, "%s (%s)" % (imp_type, val), cbx_obj.id_)) # note for display in UI