def obj_create(self, bundle, **kwargs): """ Handles creating Indicators through the API. :param bundle: Bundle containing the information to create the Indicator. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ analyst = bundle.request.user.username value = bundle.data.get('value', None) ctype = bundle.data.get('type', None) threat_type = bundle.data.get('threat_type', None) attack_type = bundle.data.get('attack_type', None) source = bundle.data.get('source', None) reference = bundle.data.get('reference', None) method = bundle.data.get('method', None) add_domain = bundle.data.get('add_domain', False) add_relationship = bundle.data.get('add_relationship', False) campaign = bundle.data.get('campaign', None) campaign_confidence = bundle.data.get('confidence', None) confidence = bundle.data.get('indicator_confidence', None) description = bundle.data.get('description', None) impact = bundle.data.get('indicator_impact', None) bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) result = handle_indicator_ind(value, source, ctype, threat_type, attack_type, analyst, method, reference, add_domain, add_relationship, campaign, campaign_confidence, confidence, description, impact, bucket_list, ticket) content = {'return_code': 0, 'type': 'Indicator'} if result.get('message'): content['message'] = result.get('message') if result.get('objectid'): url = reverse('api_dispatch_detail', kwargs={'resource_name': 'indicators', 'api_name': 'v1', 'pk': result.get('objectid')}) content['id'] = result.get('objectid') content['url'] = url if not result['success']: content['return_code'] = 1 self.crits_response(content)
def import_object(request, type_, id_): setup_access() if type_ == "Threat Descriptors": obj = ThreatDescriptor(id=id_) obj.details( fields=[f for f in ThreatDescriptor._default_fields if f not in (td.PRIVACY_MEMBERS, td.SUBMITTER_COUNT, td.METADATA)] ) itype = getattr(IndicatorTypes, obj.get(td.TYPE)) ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE)) results = handle_indicator_ind( obj.get(td.RAW_INDICATOR), "ThreatExchange", itype, ithreat_type, None, request.user.username, method="ThreatExchange Service", reference="id: %s, owner: %s, share_level: %s" % (obj.get(td.ID), obj.get(td.OWNER)['name'], obj.get(td.SHARE_LEVEL)), add_domain=True, add_relationship=True, confidence=build_ci(obj.get(td.CONFIDENCE)), description=obj.get(td.DESCRIPTION) ) return results elif type_ == "Malware Analyses": obj = Malware(id=id_) obj.details( fields=[f for f in Malware._fields if f not in (td.PRIVACY_MEMBERS, td.METADATA)] ) filename = obj.get(m.MD5) try: data = obj.rf except: data = None results = handle_file( filename, data, "ThreatExchange", method="ThreatExchange Service", reference="id: %s, share_level: %s" % (obj.get(td.ID), obj.get(td.SHARE_LEVEL)), user=request.user.username, md5_digest = obj.get(m.MD5), sha1_digest = obj.get(m.SHA1), sha256_digest = obj.get(m.SHA256), size = obj.get(m.SAMPLE_SIZE), mimetype = obj.get(m.SAMPLE_TYPE), ) return {'success': True, 'md5': results} else: return {'success': False, 'message': "Invalid Type"} return {'success': True}
def parse_indicators(self, indicators): """ Parse list of indicators. :param indicators: List of STIX indicators. :type indicators: List of STIX indicators. """ analyst = self.source_instance.analyst for indicator in indicators: # for each STIX indicator # handled indicator-wrapped observable if getattr(indicator, 'title', ""): if "Top-Level Object" in indicator.title: self.parse_observables(indicator.observables) result = self.imported.pop( indicator.observables[0].id_, None) #result[0]: Crits type | result[1]: crits obj if result: self.imported[indicator.id_] = result continue elif "MARTI Campaign" in indicator.title: return False for observable in indicator.observables: # get each observable from indicator (expecting only 1) if not observable.object_ or not observable.object_.properties: self.failed.append( ("No valid object_properties was found!", type(obs).__name__, obs.id_)) # note for display in UI continue try: # create CRITs Indicator from observable item = observable.object_.properties obj = make_crits_object(item) ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind( value.strip(), self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True) if res['success']: self.imported[indicator.id_] = ( Indicator._meta['crits_type'], res['object']) else: self.failed.append(( res['message'], type(item).__name__, item.parent.id_)) # note for display in UI except Exception, e: # probably caused by cybox object we don't handle self.failed.append( (e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def import_object(request, type_, id_): setup_access() if type_ == "Threat Descriptors": obj = ThreatDescriptor(id=id_) obj.details(fields=[ f for f in ThreatDescriptor._default_fields if f not in (td.PRIVACY_MEMBERS, td.METADATA) ]) itype = get_mapped_itype(obj.get(td.TYPE)) if itype is None: return { 'success': False, 'message': "Descriptor type is not supported by CRITs" } ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE)) results = handle_indicator_ind( obj.get(td.RAW_INDICATOR), "ThreatExchange", itype, ithreat_type, None, request.user.username, method="ThreatExchange Service", reference="id: %s, owner: %s, share_level: %s" % (obj.get( td.ID), obj.get(td.OWNER)['name'], obj.get(td.SHARE_LEVEL)), add_domain=True, add_relationship=True, confidence=build_ci(obj.get(td.CONFIDENCE)), description=obj.get(td.DESCRIPTION)) return results elif type_ == "Malware Analyses": obj = Malware(id=id_) obj.details( fields=[f for f in Malware._fields if f not in (m.METADATA)]) filename = obj.get(m.MD5) try: data = obj.rf except: data = None results = handle_file( filename, data, "ThreatExchange", method="ThreatExchange Service", reference="id: %s, share_level: %s" % (obj.get(td.ID), obj.get(td.SHARE_LEVEL)), user=request.user.username, md5_digest=obj.get(m.MD5), sha1_digest=obj.get(m.SHA1), sha256_digest=obj.get(m.SHA256), size=obj.get(m.SAMPLE_SIZE), mimetype=obj.get(m.SAMPLE_TYPE), ) return {'success': True, 'md5': results} else: return {'success': False, 'message': "Invalid Type"} return {'success': True}
def parse_indicators(self, indicators): """ Parse list of indicators. :param indicators: List of STIX indicators. :type indicators: List of STIX indicators. """ analyst = self.source_instance.analyst for indicator in indicators: # for each STIX indicator # store relationships for rel in getattr(indicator, 'related_indicators', ()): self.relationships.append((indicator.id_, rel.relationship.value, rel.item.idref, rel.confidence.value.value)) # handled indicator-wrapped observable if getattr(indicator, 'title', ""): if "Top-Level Object" in indicator.title: self.parse_observables(indicator.observables) result = self.imported.pop(indicator.observables[0].id_, None) if result: self.imported[indicator.id_] = result continue for observable in indicator.observables: # get each observable from indicator (expecting only 1) if not observable.object_ or not observable.object_.properties: self.failed.append(("No valid object_properties was found!", type(obs).__name__, obs.id_)) # note for display in UI continue try: # create CRITs Indicator from observable item = observable.object_.properties obj = make_crits_object(item) ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind(value.strip(), self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True) if res['success']: self.imported[indicator.id_] = (Indicator._meta['crits_type'], res['object']) else: self.failed.append((res['message'], type(item).__name__, item.parent.id_)) # note for display in UI except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def parse_indicators(self, indicators): """ Parse list of indicators. :param indicators: List of STIX indicators. :type indicators: List of STIX indicators. """ analyst = self.source_instance.analyst for indicator in indicators: # for each STIX indicator # store relationships for rel in getattr(indicator, 'related_indicators', ()): self.relationships.append( (indicator.id_, rel.relationship.value, rel.item.idref, rel.confidence.value.value)) # handled indicator-wrapped observable if getattr(indicator, 'title', ""): if "Top-Level Object" in indicator.title: self.parse_observables(indicator.observables) result = self.imported.pop(indicator.observables[0].id_, None) if result: self.imported[indicator.id_] = result continue for observable in indicator.observables: # get each observable from indicator (expecting only 1) try: # create CRITs Indicator from observable item = observable.object_.properties obj = make_crits_object(item) if obj.name and obj.name != obj.object_type: ind_type = "%s - %s" % (obj.object_type, obj.name) else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind(value.strip(), self.source, ind_type, analyst, add_domain=True, add_relationship=True) if res['success']: self.imported[indicator.id_] = ( Indicator._meta['crits_type'], res['object']) else: self.failed.append(( res['message'], type(item).__name__, item.parent.id_)) # note for display in UI except Exception, e: # probably caused by cybox object we don't handle self.failed.append( (e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def obj_create(self, bundle, **kwargs): """ Handles creating Indicators through the API. :param bundle: Bundle containing the information to create the Indicator. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ analyst = bundle.request.user.username value = bundle.data.get('value', None) ctype = bundle.data.get('type', None) threat_type = bundle.data.get('threat_type', None) attack_type = bundle.data.get('attack_type', None) source = bundle.data.get('source', None) reference = bundle.data.get('reference', None) method = bundle.data.get('method', None) add_domain = bundle.data.get('add_domain', False) add_relationship = bundle.data.get('add_relationship', False) campaign = bundle.data.get('campaign', None) campaign_confidence = bundle.data.get('confidence', None) confidence = bundle.data.get('indicator_confidence', None) description = bundle.data.get('description', None) impact = bundle.data.get('indicator_impact', None) bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) result = handle_indicator_ind(value, source, ctype, threat_type, attack_type, analyst, method, reference, add_domain, add_relationship, campaign, campaign_confidence, confidence, description, impact, bucket_list, ticket) content = {'return_code': 0, 'type': 'Indicator'} if result.get('message'): content['message'] = result.get('message') if result.get('objectid'): url = reverse('api_dispatch_detail', kwargs={ 'resource_name': 'indicators', 'api_name': 'v1', 'pk': result.get('objectid') }) content['id'] = result.get('objectid') content['url'] = url if not result['success']: content['return_code'] = 1 self.crits_response(content)
def parse_indicators(self, indicators): """ Parse list of indicators. :param indicators: List of STIX indicators. :type indicators: List of STIX indicators. """ analyst = self.source_instance.analyst for indicator in indicators: # for each STIX indicator # store relationships for rel in getattr(indicator, "related_indicators", ()): self.relationships.append( (indicator.id_, rel.relationship.value, rel.item.idref, rel.confidence.value.value) ) # handled indicator-wrapped observable if getattr(indicator, "title", ""): if "Top-Level Object" in indicator.title: self.parse_observables(indicator.observables) result = self.imported.pop(indicator.observables[0].id_, None) if result: self.imported[indicator.id_] = result continue for observable in indicator.observables: # get each observable from indicator (expecting only 1) try: # create CRITs Indicator from observable item = observable.object_.properties obj = make_crits_object(item) if obj.name and obj.name != obj.object_type: ind_type = "%s - %s" % (obj.object_type, obj.name) else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind( value.strip(), self.source, ind_type, analyst, add_domain=True, add_relationship=True ) if res["success"]: self.imported[indicator.id_] = (Indicator._meta["crits_type"], res["object"]) else: self.failed.append( (res["message"], type(item).__name__, item.parent.id_) ) # note for display in UI except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def obj_create(self, bundle, **kwargs): """ Handles creating Indicators through the API. :param bundle: Bundle containing the information to create the Indicator. :type bundle: Tastypie Bundle object. :returns: Bundle object. :raises BadRequest: If a campaign name is not provided or creation fails. """ analyst = bundle.request.user.username value = bundle.data.get('value', None) ctype = bundle.data.get('type', None) source = bundle.data.get('source', None) reference = bundle.data.get('reference', None) method = bundle.data.get('method', None) add_domain = bundle.data.get('add_domain', False) add_relationship = bundle.data.get('add_relationship', False) campaign = bundle.data.get('campaign', None) campaign_confidence = bundle.data.get('confidence', None) confidence = bundle.data.get('indicator_confidence', None) impact = bundle.data.get('indicator_impact', None) bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) if analyst: result = handle_indicator_ind(value, source, reference, ctype, analyst, method, add_domain, add_relationship, campaign, campaign_confidence, confidence, impact, bucket_list, ticket) if not result['success']: raise BadRequest(result['message']) else: return bundle else: raise BadRequest('You must be an authenticated user!')
def parse_indicators(self, indicators): """ Parse list of indicators. :param indicators: List of STIX indicators. :type indicators: List of STIX indicators. """ analyst = self.source_instance.analyst for indicator in indicators: # for each STIX indicator for observable in indicator.observables: # get each observable from indicator (expecting only 1) try: # create CRITs Indicator from observable item = observable.object_.properties obj = make_crits_object(item) if obj.name and obj.name != obj.object_type: ind_type = "%s - %s" % (obj.object_type, obj.name) else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind(value.strip(), self.source, None, ind_type, analyst, add_domain=True, add_relationship=True) if res['success']: self.imported.append( (Indicator._meta['crits_type'], res['object'])) else: self.failed.append(( res['message'], type(item).__name__, item.parent.id_)) # note for display in UI except Exception, e: # probably caused by cybox object we don't handle self.failed.append( (e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def obj_create(self, bundle, **kwargs): """ Handles creating Indicators through the API. :param bundle: Bundle containing the information to create the Indicator. :type bundle: Tastypie Bundle object. :returns: Bundle object. :raises BadRequest: If a campaign name is not provided or creation fails. """ analyst = bundle.request.user.username value = bundle.data.get('value', None) ctype = bundle.data.get('type', None) source = bundle.data.get('source', None) reference = bundle.data.get('reference', None) method = bundle.data.get('method', None) add_domain = bundle.data.get('add_domain', False) add_relationship = bundle.data.get('add_relationship', False) campaign = bundle.data.get('campaign', None) campaign_confidence = bundle.data.get('confidence', None) confidence = bundle.data.get('indicator_confidence', None) impact = bundle.data.get('indicator_impact', None) bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) if analyst: result = handle_indicator_ind(value, source, reference, ctype, analyst, method, add_domain, add_relationship, campaign, campaign_confidence, confidence, impact, bucket_list, ticket) if not result['success']: raise BadRequest(result['message']) else: return bundle else: raise BadRequest('You must be an authenticated user!')
def parse_indicators(self, indicators): """ Parse list of indicators. :param indicators: List of STIX indicators. :type indicators: List of STIX indicators. """ analyst = self.source_instance.analyst for indicator in indicators: # for each STIX indicator for observable in indicator.observables: # get each observable from indicator (expecting only 1) try: # create CRITs Indicator from observable item = observable.object_.properties obj = make_crits_object(item) if obj.name and obj.name != obj.object_type: ind_type = "%s - %s" % (obj.object_type, obj.name) else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind( value.strip(), self.source, None, ind_type, analyst, add_domain=True, add_relationship=True, ) if res["success"]: self.imported.append((Indicator._meta["crits_type"], res["object"])) else: self.failed.append( (res["message"], type(item).__name__, item.parent.id_) ) # note for display in UI except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def create_indicator_from_object(rel_type, rel_id, ind_type, value, source_name, method, reference, analyst, request): """ Create an indicator out of this object. :param rel_type: The top-level object type this object is for. :type rel_type: str :param rel_id: The ObjectId of the top-level object. :param ind_type: The indicator type to use. :type ind_type: str :param value: The indicator value. :type value: str :param source_name: The source name for the indicator. :type source_name: str :param method: The source method for the indicator. :type method: str :param reference: The source reference for the indicator. :type reference: str :param analyst: The user creating this indicator. :type analyst: str :param request: The Django request. :type request: :class:`django.http.HttpRequest` :returns: dict with keys "success" (bool) and "message" (str) """ result = None me = class_from_id(rel_type, rel_id) if not me: result = {'success': False, 'message': "Could not find %s" % rel_type} elif value == None or value.strip() == "": result = {'success': False, 'message': "Can't create indicator with an empty value field"} elif ind_type == None or ind_type.strip() == "": result = {'success': False, 'message': "Can't create indicator with an empty type field"} elif source_name == None or source_name.strip() == "": result = {'success': False, 'message': "Can't create indicator with an empty source field"} else: value = value.lower().strip() ind_type = ind_type.strip() source_name = source_name.strip() create_indicator_result = {} ind_tlist = ind_type.split(" - ") if ind_tlist[0] == ind_tlist[1]: ind_type = ind_tlist[0] from crits.indicators.handlers import handle_indicator_ind campaign = me.campaign if hasattr(me, 'campaign') else None create_indicator_result = handle_indicator_ind(value, source_name, reference, ind_type, analyst, method, add_domain=True, campaign=campaign) # Check if an error occurred, if it did then return the error result if create_indicator_result.get('success', True) == False: return create_indicator_result indicator = Indicator.objects(ind_type=ind_type, value=value).first() if not indicator: result = {'success': False, 'message': "Could not create indicator"} else: results = me.add_relationship(rel_item=indicator, rel_type="Related_To", analyst=analyst, get_rels=True) if results['success']: me.save(username=analyst) indicator.save(username=analyst) relationship= {'type': rel_type, 'value': rel_id} message = render_to_string('relationships_listing_widget.html', {'relationship': relationship, 'nohide': True, 'relationships': results['message']}, RequestContext(request)) result = {'success': True, 'message': message} else: message = "Indicator created. Could not create relationship" result = {'success': False, 'message': message} return result
def upload_indicator(request): """ Upload new indicators (individual, blob, or CSV file). :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :returns: :class:`django.http.HttpResponse` :class:`django.http.HttpResponseRedirect` """ if request.method == "POST": username = request.user.username failed_msg = '' result = None; if request.POST['svalue'] == "Upload CSV": form = UploadIndicatorCSVForm(username, request.POST, request.FILES) if form.is_valid(): result = handle_indicator_csv(request.FILES['filedata'], request.POST['source'], request.POST['reference'], "file", username, add_domain=True) if result['success']: message = {'message': ('<div>CSV added successfully! <a ' 'href="%s">Go to all indicators</a>' '.</div>' % reverse('crits.indicators.views.indicators_listing'))} else: failed_msg = '<div>%s</div>' % result['message'] if request.POST['svalue'] == "Upload Text": form = UploadIndicatorTextForm(username, request.POST) if form.is_valid(): result = handle_indicator_csv(request.POST['data'], request.POST['source'], request.POST['reference'], "ti", username, add_domain=True) if result['success']: message = {'message': ('<div>Indicators added successfully! ' '<a href="%s">Go to all indicators</a>' '.</div>' % reverse('crits.indicators.views.indicators_listing'))} else: failed_msg = '<div>%s</div>' % result['message'] if request.POST['svalue'] == "Upload Indicator": all_ind_type_choices = [(c[0], c[0], {'datatype': c[1].keys()[0], 'datatype_value': c[1].values()[0]}) for c in get_object_types(active=False, query={'datatype.file':{'$exists':0}})] form = UploadIndicatorForm(username, all_ind_type_choices, request.POST) if form.is_valid(): if request.POST['indicator_type'] == "URI - URL" and "://" not in request.POST['value'].split('.')[0]: result = {"success" : False, "message" : "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"} else: result = handle_indicator_ind(request.POST['value'], request.POST['source'], request.POST['reference'], request.POST['indicator_type'], username, add_domain=True, campaign=request.POST['campaign'], campaign_confidence=request.POST['campaign_confidence'], confidence=request.POST['confidence'], impact=request.POST['impact'], bucket_list=request.POST[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=request.POST[form_consts.Common.TICKET_VARIABLE_NAME]) if result['success']: indicator_link = '<a href=\"%s\">Go to this indicator</a> or <a href="%s">all indicators</a>.</div>' % (reverse('crits.indicators.views.indicator', args=[result['objectid']]), reverse('crits.indicators.views.indicators_listing')); if result.get('is_new_indicator', False) == False: message = {'message': ('<div>Warning: Updated indicator since indicator already exists! ' + indicator_link)} else: message = {'message': ('<div>Indicator added successfully! ' + indicator_link)} else: failed_msg = result['message'] if result == None or not result['success']: failed_msg += ('<a href="%s">Go to all indicators</a>' '.</div>' % reverse('crits.indicators.views.indicators_listing')) message = {'message': failed_msg, 'form': form.as_table()} elif result != None: message['success'] = result['success'] if request.is_ajax(): return HttpResponse(json.dumps(message), mimetype="application/json") else: #file upload return render_to_response('file_upload_response.html', {'response': json.dumps(message)}, RequestContext(request))
def add_object(type_, id_, object_type, source, method, reference, tlp, user, value=None, file_=None, add_indicator=False, get_objects=True, tlo=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}, **kwargs): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param id_: The ObjectId of the top-level object. :type id_: str :param object_type: The type of the ObjectType being added. :type object_type: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param user: The user adding this object. :type user: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_objects: bool :param tlo: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_sort_relationships: Return all relationships and meta, sorted :type is_sort_relationships: bool :param is_validate_only: Validate, but do not add to TLO. :type is_validate_only: bool :param is_validate_locally: Validate, but do not add b/c there is no TLO. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ # if object_type is a validated indicator type, then validate value if value: from crits.indicators.handlers import validate_indicator_value (value, error) = validate_indicator_value(value, object_type) if error: return {"success": False, "message": error} if is_validate_locally: # no TLO provided return {"success": True} if not tlo: if type_ and id_: tlo = class_from_id(type_, id_) if not tlo: return {'success': False, 'message': "Failed to find TLO"} try: if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename ret = tlo.add_object(object_type, value, source, method, reference, user) if not ret['success']: msg = '%s! [Type: "%s"][Value: "%s"]' return {"success": False, "message": msg % (ret['message'], object_type, value)} else: results = {'success': True} if not is_validate_only: # save the object tlo.update(add_to_set__obj=ret['object']) results['message'] = "Object added successfully" if file_: # do we have a pcap? if detect_pcap(data): handle_pcap_file(filename, data, source, user=user, related_id=id_, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and not is_validate_only: campaign = tlo.campaign if hasattr(tlo, 'campaign') else None from crits.indicators.handlers import handle_indicator_ind ind_res = handle_indicator_ind(value, source, object_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, source_method=method, source_reference=reference, source_tlp=tlp, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: forge_relationship(class_=tlo, right_class=ind_res['object'], rel_type=RelationshipTypes.RELATED_TO, user=user) else: msg = "Object added, but failed to add Indicator.<br>Error: %s" results['message'] = msg % ind_res.get('message') if is_sort_relationships == True: results['relationships'] = tlo.sort_relationships(user, meta=True) if get_objects: results['objects'] = tlo.sort_objects() results['id'] = str(tlo.id) return results except ValidationError as e: return {'success': False, 'message': str(e)}
def add_object(type_, id_, object_type, source, method, reference, tlp, user, value=None, file_=None, add_indicator=False, get_objects=True, tlo=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}, **kwargs): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param id_: The ObjectId of the top-level object. :type id_: str :param object_type: The type of the ObjectType being added. :type object_type: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param user: The user adding this object. :type user: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_objects: bool :param tlo: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_sort_relationships: Return all relationships and meta, sorted :type is_sort_relationships: bool :param is_validate_only: Validate, but do not add to TLO. :type is_validate_only: bool :param is_validate_locally: Validate, but do not add b/c there is no TLO. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ # if object_type is a validated indicator type, then validate value if value: from crits.indicators.handlers import validate_indicator_value (value, error) = validate_indicator_value(value, object_type) if error: return {"success": False, "message": error} if is_validate_locally: # no TLO provided return {"success": True} if not tlo: if type_ and id_: tlo = class_from_id(type_, id_) if not tlo: return {'success': False, 'message': "Failed to find TLO"} try: if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename ret = tlo.add_object(object_type, value, source, method, reference, user) if not ret['success']: msg = '%s! [Type: "%s"][Value: "%s"]' return { "success": False, "message": msg % (ret['message'], object_type, value) } else: results = {'success': True} if not is_validate_only: # save the object tlo.update(add_to_set__obj=ret['object']) results['message'] = "Object added successfully" if file_: # do we have a pcap? if detect_pcap(data): handle_pcap_file(filename, data, source, user=user, related_id=id_, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and not is_validate_only: campaign = tlo.campaign if hasattr(tlo, 'campaign') else None from crits.indicators.handlers import handle_indicator_ind ind_res = handle_indicator_ind(value, source, object_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, source_method=method, source_reference=reference, source_tlp=tlp, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: forge_relationship(class_=tlo, right_class=ind_res['object'], rel_type=RelationshipTypes.RELATED_TO, user=user) else: msg = "Object added, but failed to add Indicator.<br>Error: %s" results['message'] = msg % ind_res.get('message') if is_sort_relationships == True: results['relationships'] = tlo.sort_relationships(user, meta=True) if get_objects: results['objects'] = tlo.sort_objects() results['id'] = str(tlo.id) return results except ValidationError as e: return {'success': False, 'message': str(e)}
def ip_add_update(ip_address, ip_type, source=None, source_method='', source_reference='', source_tlp=None, campaign=None, confidence='low', user=None, is_add_indicator=False, indicator_reference='', bucket_list=None, ticket=None, is_validate_only=False, cache={}, related_id=None, related_type=None, relationship_type=None, description=''): """ Add/update an IP address. :param ip_address: The IP to add/update. :type ip_address: str :param ip_type: The type of IP this is. :type ip_type: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param campaign: A campaign to attribute to this IP address. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param user: The user adding/updating this IP. :type user: str :param is_add_indicator: Also add an Indicator for this IP. :type is_add_indicator: bool :param indicator_reference: Reference for the indicator. :type indicator_reference: str :param bucket_list: Buckets to assign to this IP. :type bucket_list: str :param ticket: Ticket to assign to this IP. :type ticket: str :param is_validate_only: Only validate, do not add/update. :type is_validate_only: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_type: str :param relationship_type: Type of relationship to create. :type relationship_type: str :param description: A description for this IP :type description: str :returns: dict with keys: "success" (boolean), "message" (str), "object" (if successful) :class:`crits.ips.ip.IP` """ if not source: return {"success": False, "message": "Missing source information."} source_name = source (ip_address, error) = validate_and_normalize_ip(ip_address, ip_type) if error: return {"success": False, "message": error} retVal = {} is_item_new = False ip_object = None cached_results = cache.get(form_consts.IP.CACHED_RESULTS) if cached_results != None: ip_object = cached_results.get(ip_address) else: ip_object = IP.objects(ip=ip_address).first() if not ip_object: ip_object = IP() ip_object.ip = ip_address ip_object.ip_type = ip_type is_item_new = True if cached_results != None: cached_results[ip_address] = ip_object if not ip_object.description: ip_object.description = description or '' elif ip_object.description != description: ip_object.description += "\n" + (description or '') if isinstance(source_name, basestring): if user.check_source_write(source): source = [ create_embedded_source(source, reference=source_reference, method=source_method, tlp=source_tlp, analyst=user.username) ] else: return { "success": False, "message": "User does not have permission to add object \ using source %s." % source } if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user.username) campaign = [c] if campaign: for camp in campaign: ip_object.add_campaign(camp) if source: for s in source: ip_object.add_source(s) else: return {"success": False, "message": "Missing source information."} if bucket_list: ip_object.add_bucket_list(bucket_list, user.username) if ticket: ip_object.add_ticket(ticket, user.username) related_obj = None if related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip]) if is_validate_only == False: ip_object.save(username=user.username) #set the URL for viewing the new data if is_item_new == True: retVal['message'] = ('Success! Click here to view the new IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) else: message = ('Updated existing IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message elif is_validate_only == True: if ip_object.id != None and is_item_new == False: message = ('Warning: IP already exists: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message if is_add_indicator: from crits.indicators.handlers import handle_indicator_ind result = handle_indicator_ind(ip_address, source_name, ip_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, source_method=source_method, source_reference=indicator_reference, source_tlp=source_tlp, add_domain=False, add_relationship=True, bucket_list=bucket_list, ticket=ticket, cache=cache) if related_obj and ip_object and relationship_type: relationship_type = RelationshipTypes.inverse( relationship=relationship_type) ip_object.add_relationship(related_obj, relationship_type, analyst=user.username, get_rels=False) ip_object.save(username=user.username) # run ip triage if is_item_new and is_validate_only == False: ip_object.reload() run_triage(ip_object, user) retVal['success'] = True retVal['object'] = ip_object return retVal
def obj_create(self, bundle, **kwargs): """ Handles creating Indicators through the API. :param bundle: Bundle containing the information to create the Indicator. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ user = bundle.request.user value = bundle.data.get('value', None) ctype = bundle.data.get('type', None) threat_type = bundle.data.get('threat_type', None) attack_type = bundle.data.get('attack_type', None) source = bundle.data.get('source', None) status = bundle.data.get('status', None) reference = bundle.data.get('reference', None) method = bundle.data.get('method', None) tlp = bundle.data.get('tlp', 'amber') add_domain = bundle.data.get('add_domain', False) add_relationship = bundle.data.get('add_relationship', False) campaign = bundle.data.get('campaign', None) campaign_confidence = bundle.data.get('confidence', None) confidence = bundle.data.get('indicator_confidence', None) description = bundle.data.get('description', None) impact = bundle.data.get('indicator_impact', None) bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) if user.has_access_to(IndicatorACL.WRITE): result = handle_indicator_ind( value, source, ctype, threat_type, attack_type, user, status=status, source_method=method, source_reference=reference, source_tlp=tlp, add_domain=add_domain, add_relationship=add_relationship, campaign=campaign, campaign_confidence=campaign_confidence, confidence=confidence, description=description, impact=impact, bucket_list=bucket_list, ticket=ticket) else: result = { 'success': False, 'message': 'User does not have permission to create Object.' } content = {'return_code': 0, 'type': 'Indicator'} if result.get('message'): content['message'] = result.get('message') if result.get('objectid'): url = reverse('api_dispatch_detail', kwargs={ 'resource_name': 'indicators', 'api_name': 'v1', 'pk': result.get('objectid') }) content['id'] = result.get('objectid') content['url'] = url if not result['success']: content['return_code'] = 1 self.crits_response(content)
def upload_indicator(request): """ Upload new indicators (individual, blob, or CSV file). :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :returns: :class:`django.http.HttpResponse` :class:`django.http.HttpResponseRedirect` """ if request.method == "POST": username = request.user.username failed_msg = '' result = None if request.POST['svalue'] == "Upload CSV": form = UploadIndicatorCSVForm(username, request.POST, request.FILES) if form.is_valid(): result = handle_indicator_csv(request.FILES['filedata'], request.POST['source'], request.POST['reference'], "file", username, add_domain=True) if result['success']: message = { 'message': ('<div>CSV added successfully! <a ' 'href="%s">Go to all indicators</a>' '.</div>' % reverse('crits.indicators.views.indicators_listing')) } else: failed_msg = '<div>%s</div>' % result['message'] if request.POST['svalue'] == "Upload Text": form = UploadIndicatorTextForm(username, request.POST) if form.is_valid(): result = handle_indicator_csv(request.POST['data'], request.POST['source'], request.POST['reference'], "ti", username, add_domain=True) if result['success']: message = { 'message': ('<div>Indicators added successfully! ' '<a href="%s">Go to all indicators</a>' '.</div>' % reverse('crits.indicators.views.indicators_listing')) } else: failed_msg = '<div>%s</div>' % result['message'] if request.POST['svalue'] == "Upload Indicator": all_ind_type_choices = [(c[0], c[0], { 'datatype': c[1].keys()[0], 'datatype_value': c[1].values()[0] }) for c in get_object_types( active=False, query={'datatype.file': { '$exists': 0 }})] form = UploadIndicatorForm(username, all_ind_type_choices, request.POST) if form.is_valid(): if request.POST[ 'indicator_type'] == "URI - URL" and "://" not in request.POST[ 'value'].split('.')[0]: result = { "success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)" } else: result = handle_indicator_ind( request.POST['value'], request.POST['source'], request.POST['reference'], request.POST['indicator_type'], username, add_domain=True, campaign=request.POST['campaign'], campaign_confidence=request. POST['campaign_confidence'], confidence=request.POST['confidence'], impact=request.POST['impact'], bucket_list=request.POST[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=request.POST[ form_consts.Common.TICKET_VARIABLE_NAME]) if result['success']: indicator_link = '<a href=\"%s\">Go to this indicator</a> or <a href="%s">all indicators</a>.</div>' % ( reverse('crits.indicators.views.indicator', args=[result['objectid']]), reverse('crits.indicators.views.indicators_listing')) if result.get('is_new_indicator', False) == False: message = { 'message': ('<div>Warning: Updated indicator since indicator already exists! ' + indicator_link) } else: message = { 'message': ('<div>Indicator added successfully! ' + indicator_link) } else: failed_msg = result['message'] if result == None or not result['success']: failed_msg += ( '<a href="%s">Go to all indicators</a>' '.</div>' % reverse('crits.indicators.views.indicators_listing')) message = {'message': failed_msg, 'form': form.as_table()} elif result != None: message['success'] = result['success'] if request.is_ajax(): return HttpResponse(json.dumps(message), mimetype="application/json") else: #file upload return render_to_response('file_upload_response.html', {'response': json.dumps(message)}, RequestContext(request))
def parse_result(self, result_extract, acl_write, md5_parent): stream_md5 = None if type(result_extract) is dict: #make reccursion extract each file embbed if 'FileMD5' in result_extract and result_extract['FileMD5']: tmp_dict = {} b_yara = False b_ioc = False #extract info no_info = [ 'ExtractInfo', 'ContainedObjects', 'Yara', 'PathFile', 'FileMD5', 'RootFileType', 'TempDirExtract' ] for key, value in result_extract.iteritems(): if not key in no_info: self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Info', key, {'value': str(value)}) #extract yara match if result_extract['Yara']: for item_v in result_extract['Yara']: for key, value in item_v.iteritems(): self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Signatures yara matched', key, value) b_yara = True #extract IOC if result_extract['ExtractInfo']: for item_v in result_extract['ExtractInfo']: for key, value in item_v.iteritems(): self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Extract potential IOC', key, {'value': str(value)}) b_ioc = True #add_sample if 'PathFile' in result_extract and type( result_extract['PathFile']) is list and len( result_extract['PathFile']) > 0: if os.path.isfile(str(result_extract['PathFile'][0])): with open(str(result_extract['PathFile'][0]), 'r') as content_file_tmp: content_tmp = content_file_tmp.read() stream_md5 = hashlib.md5(content_tmp).hexdigest() name = str(stream_md5).decode('ascii', errors='ignore') id_ = Sample.objects(md5=stream_md5).only('id').first() if id_: self._info( 'Add relationship with sample existed:' + str(stream_md5)) #make relationship id_.add_relationship( rel_item=self.obj, rel_type=RelationshipTypes.CONTAINED_WITHIN, rel_date=datetime.now(), analyst=self.current_task.user.username) else: if acl_write and ( self.config['import_file'] or (self.config['import_file_yara'] and b_yara) or (self.config['import_file_ioc'] and b_ioc)): obj_parent = None if md5_parent: obj_parent = Sample.objects( md5=md5_parent).only('id').first() if not obj_parent: sample = handle_file( name, content_tmp, self.obj.source, related_id=str(self.obj.id), related_type=str( self.obj._meta['crits_type']), campaign=self.obj.campaign, source_method=self.name, relationship=RelationshipTypes. CONTAINED_WITHIN, user=self.current_task.user) else: sample = handle_file( name, content_tmp, obj_parent.source, related_id=str(obj_parent.id), related_type=str( obj_parent._meta['crits_type']), campaign=obj_parent.campaign, source_method=self.name, relationship=RelationshipTypes. CONTAINED_WITHIN, user=self.current_task.user) self._info('Add sample ' + str(stream_md5)) else: #add IOC if not add sample if self.current_task.user.has_access_to( IndicatorACL.WRITE) and b_yara: res = handle_indicator_ind( stream_md5, self.obj.source, IndicatorTypes.MD5, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, self.current_task.user, add_relationship=True, source_method=self.name, campaign=self.obj.campaign, description='Extracted by service ' + self.name) self._info('Add indicator md5:' + str(stream_md5) + ' -- id: ' + str(res)) #contains file if 'ContainedObjects' in result_extract and type( result_extract['ContainedObjects'] ) is list and result_extract['ContainedObjects']: for item_v in result_extract['ContainedObjects']: if item_v['FileMD5'] and item_v['FileType'] and item_v[ 'FileSize']: #search if file exist id_ = Sample.objects( md5=str(item_v['FileMD5'])).only('id').first() sample_exist = False ioc_exist = False if id_: sample_exist = True id_ = Indicator.objects( value=str(item_v['FileMD5'])).only('id').first() if id_: ioc_exist = True self._add_result( 'File: ' + result_extract['FileMD5'] + ' - Contains md5 files', item_v['FileMD5'], { 'type': str(item_v['FileType']), 'size': str(item_v['FileSize']), 'Exists Sample': str(sample_exist), 'Exists IOC md5': str(ioc_exist) }) for item_v in result_extract['ContainedObjects']: #re do loop for best display result parse_result(self, item_v, acl_write, stream_md5)
def upload_indicator(request): """ Upload new indicators (individual, blob, or CSV file). :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :returns: :class:`django.http.HttpResponse` :class:`django.http.HttpResponseRedirect` """ if request.method == "POST": username = request.user.username failed_msg = "" result = None if request.POST["svalue"] == "Upload CSV": form = UploadIndicatorCSVForm(username, request.POST, request.FILES) if form.is_valid(): result = handle_indicator_csv( request.FILES["filedata"], request.POST["source"], request.POST["method"], request.POST["reference"], "file", username, add_domain=True, related_id=request.POST["related_id"], related_type=request.POST["related_type"], relationship_type=request.POST["relationship_type"], ) if result["success"]: message = { "message": ( '<div>%s <a href="%s">Go to all' " indicators</a></div>" % (result["message"], reverse("crits.indicators.views.indicators_listing")) ) } else: failed_msg = "<div>%s</div>" % result["message"] if request.POST["svalue"] == "Upload Text": form = UploadIndicatorTextForm(username, request.POST) if form.is_valid(): result = handle_indicator_csv( request.POST["data"], request.POST["source"], request.POST["method"], request.POST["reference"], "ti", username, add_domain=True, related_id=request.POST["related_id"], related_type=request.POST["related_type"], relationship_type=request.POST["relationship_type"], ) if result["success"]: message = { "message": ( '<div>%s <a href="%s">Go to all' " indicators</a></div>" % (result["message"], reverse("crits.indicators.views.indicators_listing")) ) } else: failed_msg = "<div>%s</div>" % result["message"] if request.POST["svalue"] == "Upload Indicator": form = UploadIndicatorForm(username, request.POST) if form.is_valid(): result = handle_indicator_ind( request.POST["value"], request.POST["source"], request.POST["indicator_type"], request.POST["threat_type"], request.POST["attack_type"], username, status=request.POST["status"], method=request.POST["method"], reference=request.POST["reference"], add_domain=True, description=request.POST["description"], campaign=request.POST["campaign"], campaign_confidence=request.POST["campaign_confidence"], confidence=request.POST["confidence"], impact=request.POST["impact"], bucket_list=request.POST[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=request.POST[form_consts.Common.TICKET_VARIABLE_NAME], related_id=request.POST["related_id"], related_type=request.POST["related_type"], relationship_type=request.POST["relationship_type"], ) if result["success"]: indicator_link = ( ' - <a href="%s">Go to this ' 'indicator</a> or <a href="%s">all ' "indicators</a>.</div>" ) % ( reverse("crits.indicators.views.indicator", args=[result["objectid"]]), reverse("crits.indicators.views.indicators_listing"), ) if result.get("is_new_indicator", False) == False: message = {"message": ("<div>Warning: Updated existing" " Indicator!" + indicator_link)} else: message = {"message": ("<div>Indicator added " "successfully!" + indicator_link)} else: failed_msg = result["message"] + " - " if result == None or not result["success"]: failed_msg += '<a href="%s"> Go to all indicators</a></div>' % reverse( "crits.indicators.views.indicators_listing" ) message = {"message": failed_msg, "form": form.as_table()} elif result != None: message["success"] = result["success"] if request.is_ajax(): return HttpResponse(json.dumps(message), content_type="application/json") else: # file upload return render_to_response( "file_upload_response.html", {"response": json.dumps(message)}, RequestContext(request) )
def ip_add_update(ip_address, ip_type, source=None, source_method=None, source_reference=None, campaign=None, confidence='low', analyst=None, is_add_indicator=False, indicator_reference=None, bucket_list=None, ticket=None, is_validate_only=False, cache={}): """ Add/update an IP address. :param ip_address: The IP to add/update. :type ip_address: str :param ip_type: The type of IP this is. :type ip_type: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param campaign: A campaign to attribute to this IP address. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param analyst: The user adding/updating this IP. :type analyst: str :param is_add_indicator: Also add an Indicator for this IP. :type is_add_indicator: bool :param indicator_reference: Reference for the indicator. :type indicator_reference: str :param bucket_list: Buckets to assign to this IP. :type bucket_list: str :param ticket: Ticket to assign to this IP. :type ticket: str :param is_validate_only: Only validate, do not add/update. :type is_validate_only: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "object" (if successful) :class:`crits.ips.ip.IP` """ retVal = {} is_item_new = False ip_object = None cached_results = cache.get(form_consts.IP.CACHED_RESULTS) if cached_results != None: ip_object = cached_results.get(ip_address) else: ip_object = IP.objects(ip=ip_address).first() if not ip_object: ip_object = IP() ip_object.ip = ip_address ip_object.ip_type = ip_type is_item_new = True if cached_results != None: cached_results[ip_address] = ip_object if isinstance(source, basestring): source = [create_embedded_source(source, reference=source_reference, method=source_method, analyst=analyst)] if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst) campaign = [c] if campaign: for camp in campaign: ip_object.add_campaign(camp) if source: for s in source: ip_object.add_source(s) if bucket_list: ip_object.add_bucket_list(bucket_list, analyst) if ticket: ip_object.add_ticket(ticket, analyst) resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip]) if is_validate_only == False: ip_object.save(username=analyst) #set the URL for viewing the new data if is_item_new == True: retVal['message'] = ('Success! Click here to view the new IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) else: message = ('Updated existing IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message elif is_validate_only == True: if ip_object.id != None and is_item_new == False: message = ('Warning: IP already exists: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message if is_add_indicator: from crits.indicators.handlers import handle_indicator_ind handle_indicator_ind(ip_address, source, indicator_reference, ip_type, analyst, source_method, add_domain=False, add_relationship=True, bucket_list=bucket_list, ticket=ticket, cache=cache) # run ip triage if is_item_new and is_validate_only == False: ip_object.reload() run_triage(None, ip_object, analyst) retVal['success'] = True retVal['object'] = ip_object return retVal
def obj_create(self, bundle, **kwargs): """ Handles creating Indicators through the API. :param bundle: Bundle containing the information to create the Indicator. :type bundle: Tastypie Bundle object. :returns: HttpResponse. """ user = bundle.request.user value = bundle.data.get('value', None) ctype = bundle.data.get('type', None) threat_type = bundle.data.get('threat_type', None) attack_type = bundle.data.get('attack_type', None) source = bundle.data.get('source', None) status = bundle.data.get('status', None) reference = bundle.data.get('reference', None) method = bundle.data.get('method', None) tlp = bundle.data.get('tlp', 'amber') add_domain = bundle.data.get('add_domain', False) add_relationship = bundle.data.get('add_relationship', False) campaign = bundle.data.get('campaign', None) campaign_confidence = bundle.data.get('confidence', None) confidence = bundle.data.get('indicator_confidence', None) description = bundle.data.get('description', None) impact = bundle.data.get('indicator_impact', None) bucket_list = bundle.data.get('bucket_list', None) ticket = bundle.data.get('ticket', None) if user.has_access_to(IndicatorACL.WRITE): result = handle_indicator_ind(value, source, ctype, threat_type, attack_type, user, status=status, source_method=method, source_reference=reference, source_tlp=tlp, add_domain=add_domain, add_relationship=add_relationship, campaign=campaign, campaign_confidence=campaign_confidence, confidence=confidence, description=description, impact=impact, bucket_list=bucket_list, ticket=ticket) else: result = {'success':False, 'message':'User does not have permission to create Object.'} content = {'return_code': 0, 'type': 'Indicator'} if result.get('message'): content['message'] = result.get('message') if result.get('objectid'): url = reverse('api_dispatch_detail', kwargs={'resource_name': 'indicators', 'api_name': 'v1', 'pk': result.get('objectid')}) content['id'] = result.get('objectid') content['url'] = url if not result['success']: content['return_code'] = 1 self.crits_response(content)
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, indicator_campaign=None, indicator_campaign_confidence=None, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, related_id=oid, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) ind_res = handle_indicator_ind( value, source, reference, object_type, analyst, method=method, add_domain=True, campaign=indicator_campaign, campaign_confidence=indicator_campaign_confidence, cache=cache) if ind_res['success']: ind = ind_res['object'] # Inherit campaigns from top level item when creating # an indicator from an object if no campaigns were specified if indicator_campaign == None and ind != None: for campaign in obj.campaign: ec = EmbeddedCampaign( name=campaign.name, confidence=campaign.confidence, description="", analyst=analyst, date=datetime.datetime.now()) ind.add_campaign(ec) ind.save(username=analyst) forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results[ 'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() return results except ValidationError, e: return {'success': False, 'message': e}
def add_object(type_, id_, object_type, source, method, reference, user, value=None, file_=None, add_indicator=False, get_objects=True, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}, **kwargs): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param id_: The ObjectId of the top-level object. :type id_: str :param object_type: The type of the ObjectType being added. :type object_type: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param user: The user adding this object. :type user: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_validate_only: Validate, but do not add to TLO. :type is_validate_only: bool :param is_validate_locally: Validate, but do not add b/c there is no TLO. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if id_ == None: id_ = "" if obj == None: obj = class_from_id(type_, id_) from crits.indicators.handlers import validate_indicator_value if value is not None: (value, error) = validate_indicator_value(value, object_type) if error: return {"success": False, "message": error} if is_validate_locally: # no obj provided results['success'] = True return results if not obj: results['message'] = "TLO could not be found" results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, value, source, method, reference, user) if is_validate_only == False: obj.save(username=user) new_len = len(obj.obj) if new_len > cur_len: if not is_validate_only: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=user, related_id=id_, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind campaign = obj.campaign if hasattr(obj, 'campaign') else None ind_res = handle_indicator_ind(value, source, object_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, method, reference, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: ind = ind_res['object'] forge_relationship(class_=obj, right_class=ind, rel_type=RelationshipTypes.RELATED_TO, user=user, get_rels=is_sort_relationships) else: results['message'] = "Object was added, but failed to add Indicator." \ "<br>Error: " + ind_res.get('message') if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships(user, meta=True) else: results['relationships'] = obj.sort_relationships(user, meta=True) else: results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() results['id'] = str(obj.id) return results except ValidationError, e: return {'success': False, 'message': str(e)}
def parse_cybox_object(self, cbx_obj, description='', ind_id=None): """ Parse a CybOX object form a STIX doc. An object can contain multiple related_objects, which in turn can have their own related_objects, so this handles those recursively. :param cbx_obj: The CybOX object to parse. :type cbx_obj: A CybOX object. :param description: Parent-level (e.g. Observable) description. :type description: str :param ind_id: The ID of a parent STIX Indicator. :type ind_id: str """ # check for missing attributes if not cbx_obj or not cbx_obj.properties: if cbx_obj.idref: # just a reference, so nothing to parse return else: cbx_id = getattr(cbx_obj, 'id_', 'None') self.failed.append(("No valid object_properties was found!", "Observable (%s)" % cbx_id, cbx_id)) # note for display in UI return # Don't parse if already been parsed # This is for artifacts that are related to CybOX File Objects if cbx_obj.id_ in self.parsed: return try: # try to create CRITs object from Cybox Object analyst = self.source_instance.analyst item = cbx_obj.properties val = cbx_obj.id_ if isinstance(item, Address) and not ind_id: if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: val = str(value).strip() if self.preview: res = None else: iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(val, iptype, [self.source], analyst=analyst, is_add_indicator=True) else: res = { 'success': False, 'reason': 'No IP Type' } self.parse_res(imp_type, val, cbx_obj, res, ind_id) if (not ind_id and (isinstance(item, DomainName) or (isinstance(item, URI) and item.type_ == 'Domain Name'))): imp_type = "Domain" for val in item.value.values: if self.preview: res = None else: res = upsert_domain(str(val), [self.source], username=analyst) self.parse_res(imp_type, str(val), cbx_obj, res, ind_id) elif isinstance(item, HTTPSession): imp_type = "RawData" val = cbx_obj.id_ try: c_req = item.http_request_response[0].http_client_request hdr = c_req.http_request_header if hdr.raw_header: data = hdr.raw_header.value title = "HTTP Header from STIX: %s" % self.package.id_ method = self.source_instance.method ref = self.source_instance.reference if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="HTTP Header", tool_name="STIX", tool_version=None, method=method, reference=ref) else: imp_type = "Indicator" ind_type = "HTTP Request Header Fields - User-Agent" val = hdr.parsed_header.user_agent.value val = ','.join(val) if isinstance(val, list) else val if self.preview: res = None else: res = handle_indicator_ind( val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_relationship=True, description=description) except: msg = "Unsupported use of 'HTTPSession' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, WhoisEntry): # No sure where else to put this imp_type = "RawData" val = cbx_obj.id_ if item.remarks: data = item.remarks.value title = "WHOIS Entry from STIX: %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file( data, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="WHOIS", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) else: msg = "Unsupported use of 'WhoisEntry' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" val = cbx_obj.id_ rawdata = item.data.decode('utf-8') # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file( rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if isinstance(rel_obj.properties, Artifact): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_cert_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_NETWORK): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_pcap_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File): imp_type = "Sample" md5 = item.md5 if md5: md5 = md5.lower() val = str(item.file_name or md5) # add sha1/sha256/ssdeep once handle_file supports it size = item.size_in_bytes data = None if item.file_path: path = "File Path: " + str(item.file_path) description += "\n" + path for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_FILE): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) if not md5 and not data and val and val != "None": imp_type = "Indicator" if self.preview: res = None else: res = handle_indicator_ind( val, self.source, "Win File", IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) elif md5 or data: if self.preview: res = None else: res = handle_file(val, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, size=size, description=description) else: val = cbx_obj.id_ msg = "CybOX 'File' object has no MD5, data, or filename" res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, EmailMessage): imp_type = 'Email' id_list = [] data = {} val = cbx_obj.id_ get_attach = False data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['subject'] = str(item.header.subject) if item.header.date: data['date'] = item.header.date.value val = "Date: %s, Subject: %s" % (data.get( 'date', 'None'), data['subject']) data['message_id'] = str(item.header.message_id) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str( item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] if data.get('date'): # Email TLOs must have a date data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference if self.preview: res = None else: res = handle_email_fields(data, analyst, "STIX") self.parse_res(imp_type, val, cbx_obj, res, ind_id) if not self.preview and res.get('status'): id_list.append(cbx_obj.id_) # save ID for atchmnt rels get_attach = True else: # Can't be an Email TLO, so save fields for x, key in enumerate(data): if data[key] and data[key] != "None": if key in ('raw_header', 'raw_body'): if key == 'raw_header': title = "Email Header from STIX Email: %s" d_type = "Email Header" else: title = "Email Body from STIX Email: %s" d_type = "Email Body" imp_type = 'RawData' title = title % cbx_obj.id_ if self.preview: res = None else: res = handle_raw_data_file( data[key], self.source, analyst, description, title, d_type, "STIX", self.stix_version) self.parse_res(imp_type, title, cbx_obj, res, ind_id) elif key == 'to': imp_type = 'Target' for y, addr in enumerate(data[key]): tgt_dict = {'email_address': addr} if self.preview: res = None else: res = upsert_target(tgt_dict, analyst) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x, y) self.parse_res(imp_type, addr, tmp_obj, res, ind_id) self.ind2obj.setdefault( cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) else: imp_type = 'Indicator' if key in ('sender', 'reply_to', 'from_address'): ind_type = "Address - e-mail" elif 'ip' in key: ind_type = "Address - ipv4-addr" elif key == 'raw_body': ind_type = "Email Message" else: ind_type = "String" if self.preview: res = None else: res = handle_indicator_ind( data[key], self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x) self.parse_res(imp_type, data[key], tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) if not self.preview: # Setup relationships between all Email attributes for oid in id_list: for oid2 in id_list: if oid != oid2: self.relationships.append( (oid, RelationshipTypes.RELATED_TO, oid2, "High")) # Should check for attachments and add them here. if get_attach and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] for oid in id_list: self.relationships.append( (oid, RelationshipTypes.CONTAINS, rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" val = cbx_obj.id_ c_obj = make_crits_object(item) # Ignore what was already caught above if (ind_id or c_obj.object_type not in IPTypes.values()): ind_type = c_obj.object_type for val in [str(v).strip() for v in c_obj.value if v]: if ind_type: # handle domains mislabeled as URLs if c_obj.object_type == 'URI' and '/' not in val: ind_type = "Domain" if self.preview: res = None else: res = handle_indicator_ind( val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, "%s (%s)" % (imp_type, val), cbx_obj.id_)) # note for display in UI
def parse_observables(self, observables): """ Parse list of observables in STIX doc. :param observables: List of STIX observables. :type observables: List of STIX observables. """ analyst = self.source_instance.analyst for obs in observables: # for each STIX observable if not obs.object_ or not obs.object_.properties: self.failed.append(("No valid object_properties was found!", type(obs).__name__, obs.id_)) # note for display in UI continue try: # try to create CRITs object from observable item = obs.object_.properties if isinstance(item, Address): if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: ip = str(value).strip() iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(ip, iptype, [self.source], analyst=analyst, is_add_indicator=True) self.parse_res(imp_type, obs, res) if isinstance(item, DomainName): imp_type = "Domain" for value in item.value.values: res = upsert_domain(str(value), [self.source], username=analyst) self.parse_res(imp_type, obs, res) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" rawdata = item.data.decode('utf-8') description = "None" # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ res = handle_raw_data_file(rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, obs, res) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if isinstance(obj.properties, Artifact): data = obj.properties.data res = handle_cert_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_NETWORK): data = obj.properties.data res = handle_pcap_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File): imp_type = "Sample" filename = str(item.file_name) md5 = item.md5 data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_FILE): data = obj.properties.data res = handle_file(filename, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False) self.parse_res(imp_type, obs, res) elif isinstance(item, EmailMessage): imp_type = "Email" data = {} data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['message_id'] = str(item.header.message_id) data['subject'] = str(item.header.subject) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str(item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) data['date'] = item.header.date.value if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] res = handle_email_fields(data, analyst, "STIX") # Should check for attachments and add them here. self.parse_res(imp_type, obs, res) if res.get('status') and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] self.relationships.append((obs.id_, "Contains", rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" obj = make_crits_object(item) if obj.object_type == 'Address': # This was already caught above continue else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind(value.strip(), self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True) self.parse_res(imp_type, obs, res) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def create_indicator_from_object(rel_type, rel_id, ind_type, value, source_name, method, reference, tlp, analyst, request): """ Create an indicator out of this object. :param rel_type: The top-level object type this object is for. :type rel_type: str :param rel_id: The ObjectId of the top-level object. :param ind_type: The indicator type to use. :type ind_type: str :param value: The indicator value. :type value: str :param source_name: The source name for the indicator. :type source_name: str :param method: The source method for the indicator. :type method: str :param reference: The source reference for the indicator. :type reference: str :param analyst: The user creating this indicator. :type analyst: str :param request: The Django request. :type request: :class:`django.http.HttpRequest` :returns: dict with keys "success" (bool) and "message" (str) """ result = None me = class_from_id(rel_type, rel_id) if not me: result = {'success': False, 'message': "Could not find %s" % rel_type} elif value == None or value.strip() == "": result = { 'success': False, 'message': "Can't create indicator with an empty value field" } elif ind_type == None or ind_type.strip() == "": result = { 'success': False, 'message': "Can't create indicator with an empty type field" } elif source_name == None or source_name.strip() == "": result = { 'success': False, 'message': "Can't create indicator with an empty source field" } else: value = value.lower().strip() ind_type = ind_type.strip() source_name = source_name.strip() create_indicator_result = {} from crits.indicators.handlers import handle_indicator_ind campaign = me.campaign if hasattr(me, 'campaign') else None create_indicator_result = handle_indicator_ind( value, source_name, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, source_method=method, source_reference=reference, source_tlp=tlp, add_domain=True, campaign=campaign) # Check if an error occurred, if it did then return the error result if create_indicator_result.get('success', True) == False: return create_indicator_result indicator = Indicator.objects(ind_type=ind_type, value=value).first() if not indicator: result = { 'success': False, 'message': "Could not create indicator" } else: results = me.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst=analyst, get_rels=True) if results['success']: me.save(username=analyst) relationship = {'type': rel_type, 'value': rel_id} message = render_to_string( 'relationships_listing_widget.html', { 'relationship': relationship, 'nohide': True, 'relationships': results['message'] }, RequestContext(request)) result = {'success': True, 'message': message} else: message = "Indicator created. Could not create relationship" result = {'success': False, 'message': message} return result
def upload_indicator(request): """ Upload new indicators (individual, blob, or CSV file). :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :returns: :class:`django.http.HttpResponse` :class:`django.http.HttpResponseRedirect` """ if request.method == "POST": username = request.user.username failed_msg = '' result = None if request.POST['svalue'] == "Upload CSV": form = UploadIndicatorCSVForm( username, request.POST, request.FILES) if form.is_valid(): result = handle_indicator_csv(request.FILES['filedata'], request.POST['source'], request.POST['method'], request.POST['reference'], "file", username, add_domain=True) if result['success']: message = {'message': ('<div>%s <a href="%s">Go to all' ' indicators</a></div>' % (result['message'], reverse('crits.indicators.views.indicators_listing')))} else: failed_msg = '<div>%s</div>' % result['message'] if request.POST['svalue'] == "Upload Text": form = UploadIndicatorTextForm(username, request.POST) if form.is_valid(): result = handle_indicator_csv(request.POST['data'], request.POST['source'], request.POST['method'], request.POST['reference'], "ti", username, add_domain=True) if result['success']: message = {'message': ('<div>%s <a href="%s">Go to all' ' indicators</a></div>' % (result['message'], reverse('crits.indicators.views.indicators_listing')))} else: failed_msg = '<div>%s</div>' % result['message'] if request.POST['svalue'] == "Upload Indicator": form = UploadIndicatorForm(username, request.POST) if form.is_valid(): result = handle_indicator_ind( request.POST['value'], request.POST['source'], request.POST['indicator_type'], request.POST['threat_type'], request.POST['attack_type'], username, request.POST['method'], request.POST['reference'], add_domain=True, description=request.POST['description'], campaign=request.POST['campaign'], campaign_confidence=request.POST['campaign_confidence'], confidence=request.POST['confidence'], impact=request.POST['impact'], bucket_list=request.POST[form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=request.POST[form_consts.Common.TICKET_VARIABLE_NAME]) if result['success']: indicator_link = ((' - <a href=\"%s\">Go to this ' 'indicator</a> or <a href="%s">all ' 'indicators</a>.</div>') % (reverse('crits.indicators.views.indicator', args=[result['objectid']]), reverse('crits.indicators.views.indicators_listing'))) if result.get('is_new_indicator', False) == False: message = {'message': ('<div>Warning: Updated existing' ' Indicator!' + indicator_link)} else: message = {'message': ('<div>Indicator added ' 'successfully!' + indicator_link)} else: failed_msg = result['message'] + ' - ' if result == None or not result['success']: failed_msg += ('<a href="%s"> Go to all indicators</a></div>' % reverse('crits.indicators.views.indicators_listing')) message = {'message': failed_msg, 'form': form.as_table()} elif result != None: message['success'] = result['success'] if request.is_ajax(): return HttpResponse(json.dumps(message), mimetype="application/json") else: #file upload return render_to_response('file_upload_response.html', {'response': json.dumps(message)}, RequestContext(request))
def parse_observables(self, observables): """ Parse list of observables in STIX doc. :param observables: List of STIX observables. :type observables: List of STIX observables. """ analyst = self.source_instance.analyst for obs in observables: # for each STIX observable if obs.observable_composition: object_list = obs.observable_composition.observables else: object_list = [obs] for obs_comp in object_list: if not obs_comp.object_ or not obs_comp.object_.properties: self.failed.append( ("No valid object_properties was found!", type(obs_comp).__name__, obs_comp.id_)) # note for display in UI continue try: # try to create CRITs object from observable item = obs_comp.object_.properties if isinstance(item, Address): if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask', 'ipv6-subnet'): imp_type = "IP" for value in item.address_value.values: ip = str(value).strip() iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(ip, iptype, [self.source], analyst=analyst, id=self.package.id_) self.parse_res(imp_type, obs, res) if isinstance(item, DomainName): imp_type = "Domain" for value in item.value.values: res = upsert_domain(str(value), [self.source], username=analyst, id=self.package.id_) self.parse_res(imp_type, obs, res) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" rawdata = item.data.decode('utf-8') description = "None" # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ res = handle_raw_data_file( rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, obs, res) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if isinstance(obj.properties, Artifact): data = obj.properties.data res = handle_cert_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" description = "None" filename = str(item.file_name) data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_NETWORK): data = obj.properties.data res = handle_pcap_file(filename, data, self.source, user=analyst, description=description) self.parse_res(imp_type, obs, res) elif isinstance(item, File): imp_type = "Sample" filename = str(item.file_name) md5 = item.md5 data = None for obj in item.parent.related_objects: if (isinstance(obj.properties, Artifact) and obj.properties.type_ == Artifact.TYPE_FILE): data = obj.properties.data res = handle_file(filename, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, id=self.package.id_) self.parse_res(imp_type, obs, res) if item.extracted_features: self.parse_filenames(item.extracted_features, res['object'].id) elif isinstance(item, EmailMessage): imp_type = "Email" data = {} data['source'] = self.source.name data['source_method'] = self.source_instance.method data[ 'source_reference'] = self.source_instance.reference data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['message_id'] = str(item.header.message_id) data['subject'] = str(item.header.subject) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str( item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) data['date'] = item.header.date.value if item.header.to: data['to'] = [str(r) for r in item.header.to] if item.header.cc: data['cc'] = [str(r) for r in item.header.cc] res = handle_email_fields(data, analyst, "STIX", id=self.package.id_) # Should check for attachments and add them here. self.parse_res(imp_type, obs, res) if res.get('status') and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] self.relationships.append( (obs.id_, "Contains", rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" obj = make_crits_object(item) if obj.object_type == 'Address': # This was already caught above continue else: ind_type = obj.object_type for value in obj.value: if value and ind_type: res = handle_indicator_ind( value.strip(), self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True) self.parse_res(imp_type, obs, res) except Exception, e: # probably caused by cybox object we don't handle self.failed.append( (e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def create_indicator_from_object(rel_type, rel_id, ind_type, value, analyst, request): """ Create an indicator out of this object. :param rel_type: The top-level object type this object is for. :type rel_type: str :param rel_id: The ObjectId of the top-level object. :param ind_type: The indicator type to use. :type ind_type: str :param value: The indicator value. :type value: str :param analyst: The user creating this indicator. :type analyst: str :param request: The Django request. :type request: :class:`django.http.HttpRequest` :returns: dict with keys "success" (bool) and "message" (str) """ result = None me = class_from_id(rel_type, rel_id) if not me: result = {'success': False, 'message': "Could not find %s" % rel_type} elif value == None or value.strip() == "": result = { 'success': False, 'message': "Can't create indicator with an empty value field" } elif ind_type == None or ind_type.strip() == "": result = { 'success': False, 'message': "Can't create indicator with an empty type field" } else: value = value.lower().strip() ind_type = ind_type.strip() create_indicator_result = {} ind_tlist = ind_type.split(" - ") if ind_tlist[0] == ind_tlist[1]: ind_type = ind_tlist[0] from crits.indicators.handlers import handle_indicator_ind if hasattr(me, 'source'): create_indicator_result = handle_indicator_ind(value, me.source, '', ind_type, analyst=analyst, add_domain=True) else: # In case the top level item doesn't have sources (such as campaign)... # then just default to the user's organization create_indicator_result = handle_indicator_ind( value, get_user_organization(analyst), '', ind_type, analyst=analyst, add_domain=True) # Check if an error occurred, if it did then return the error result if create_indicator_result.get('success', True) == False: return result indicator = Indicator.objects(ind_type=ind_type, value=value).first() if not indicator: result = { 'success': False, 'message': "Could not create indicator" } else: results = me.add_relationship(rel_item=indicator, rel_type="Related_To", analyst=analyst, get_rels=True) if results['success']: me.save(username=analyst) indicator.save(username=analyst) relationship = {'type': rel_type, 'value': rel_id} message = render_to_string( 'relationships_listing_widget.html', { 'relationship': relationship, 'nohide': True, 'relationships': results['message'] }, RequestContext(request)) result = {'success': True, 'message': message} else: message = "Indicator created. Could not create relationship" result = {'success': False, 'message': message} return result
def upload_indicator(request): """ Upload new indicators (individual, blob, or CSV file). :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :returns: :class:`django.http.HttpResponse` :class:`django.http.HttpResponseRedirect` """ if request.method == "POST": username = request.user.username failed_msg = '' result = None if request.POST['svalue'] == "Upload CSV": form = UploadIndicatorCSVForm(username, request.POST, request.FILES) if form.is_valid(): result = handle_indicator_csv( request.FILES['filedata'], request.POST['source'], request.POST['method'], request.POST['reference'], "file", username, add_domain=True, related_id=request.POST['related_id'], related_type=request.POST['related_type'], relationship_type=request.POST['relationship_type']) if result['success']: message = { 'message': ('<div>%s <a href="%s">Go to all' ' indicators</a></div>' % (result['message'], reverse('crits.indicators.views.indicators_listing')) ) } else: failed_msg = '<div>%s</div>' % result['message'] if request.POST['svalue'] == "Upload Text": form = UploadIndicatorTextForm(username, request.POST) if form.is_valid(): result = handle_indicator_csv( request.POST['data'], request.POST['source'], request.POST['method'], request.POST['reference'], "ti", username, add_domain=True, related_id=request.POST['related_id'], related_type=request.POST['related_type'], relationship_type=request.POST['relationship_type']) if result['success']: message = { 'message': ('<div>%s <a href="%s">Go to all' ' indicators</a></div>' % (result['message'], reverse('crits.indicators.views.indicators_listing')) ) } else: failed_msg = '<div>%s</div>' % result['message'] if request.POST['svalue'] == "Upload Indicator": form = UploadIndicatorForm(username, request.POST) if form.is_valid(): result = handle_indicator_ind( request.POST['value'], request.POST['source'], request.POST['indicator_type'], request.POST['threat_type'], request.POST['attack_type'], username, request.POST['method'], request.POST['reference'], add_domain=True, description=request.POST['description'], campaign=request.POST['campaign'], campaign_confidence=request.POST['campaign_confidence'], confidence=request.POST['confidence'], impact=request.POST['impact'], bucket_list=request.POST[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=request.POST[ form_consts.Common.TICKET_VARIABLE_NAME], related_id=request.POST['related_id'], related_type=request.POST['related_type'], relationship_type=request.POST['relationship_type']) if result['success']: indicator_link = ( (' - <a href=\"%s\">Go to this ' 'indicator</a> or <a href="%s">all ' 'indicators</a>.</div>') % (reverse('crits.indicators.views.indicator', args=[result['objectid']]), reverse('crits.indicators.views.indicators_listing'))) if result.get('is_new_indicator', False) == False: message = { 'message': ('<div>Warning: Updated existing' ' Indicator!' + indicator_link) } else: message = { 'message': ('<div>Indicator added ' 'successfully!' + indicator_link) } else: failed_msg = result['message'] + ' - ' if result == None or not result['success']: failed_msg += ( '<a href="%s"> Go to all indicators</a></div>' % reverse('crits.indicators.views.indicators_listing')) message = {'message': failed_msg, 'form': form.as_table()} elif result != None: message['success'] = result['success'] if request.is_ajax(): return HttpResponse(json.dumps(message), content_type="application/json") else: #file upload return render_to_response('file_upload_response.html', {'response': json.dumps(message)}, RequestContext(request))
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results if name == "URL" and "://" not in value.split('.')[0]: return { "success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)" } elif object_type == "Address": if "ipv4" in name: try: validate_ipv4_address(value) except DjangoValidationError: return {"success": False, "message": "Invalid IPv4 address. "} elif "ipv6" in name: try: validate_ipv6_address(value) except DjangoValidationError: return {"success": False, "message": "Invalid IPv6 address. "} elif "cidr" in name: try: if '/' not in value: raise ValidationError("") cidr_parts = value.split('/') if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128: raise ValidationError("") if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32: raise ValidationError("") validate_ipv46_address(cidr_parts[0]) except (ValidationError, ValueError) as cidr_error: return {"success": False, "message": "Invalid CIDR address. "} try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, related_id=oid, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) campaign = obj.campaign if hasattr(obj, 'campaign') else None ind_res = handle_indicator_ind(value, source, reference, object_type, analyst, method, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: ind = ind_res['object'] forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) else: results['message'] = "Object was added, but failed to add Indicator." \ "<br>Error: " + ind_res.get('message') if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results[ 'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() results['id'] = str(obj.id) return results except ValidationError, e: return {'success': False, 'message': str(e)}
def add_object(type_, id_, object_type, source, method, reference, user, value=None, file_=None, add_indicator=False, get_objects=True, tlo=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}, **kwargs): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param id_: The ObjectId of the top-level object. :type id_: str :param object_type: The type of the ObjectType being added. :type object_type: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param user: The user adding this object. :type user: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param tlo: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_validate_only: Validate, but do not add to TLO. :type is_validate_only: bool :param is_validate_locally: Validate, but do not add b/c there is no TLO. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} obj = tlo if id_ == None: id_ = "" if obj == None: obj = class_from_id(type_, id_) from crits.indicators.handlers import validate_indicator_value if value is not None: (value, error) = validate_indicator_value(value, object_type) if error: return {"success": False, "message": error} if is_validate_locally: # no obj provided results['success'] = True return results if not obj: results['message'] = "TLO could not be found" results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, value, source, method, reference, user) if is_validate_only == False: obj.save(username=user) new_len = len(obj.obj) if new_len > cur_len: if not is_validate_only: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=user, related_id=id_, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind campaign = obj.campaign if hasattr(obj, 'campaign') else None ind_res = handle_indicator_ind(value, source, object_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, method, reference, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: ind = ind_res['object'] forge_relationship(class_=obj, right_class=ind, rel_type=RelationshipTypes.RELATED_TO, user=user, get_rels=is_sort_relationships) else: results['message'] = "Object was added, but failed to add Indicator." \ "<br>Error: " + ind_res.get('message') if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships( user, meta=True) else: results['relationships'] = obj.sort_relationships( user, meta=True) else: results[ 'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() results['id'] = str(obj.id) return results except ValidationError, e: return {'success': False, 'message': str(e)}
def import_object(request, type_, id_): setup_access() user = request.user if type_ == "Threat Descriptors": obj = ThreatDescriptor(id=id_) obj.details( fields=[f for f in ThreatDescriptor._default_fields if f not in (td.PRIVACY_MEMBERS, td.METADATA)] ) itype = get_mapped_itype(obj.get(td.TYPE)) tags = obj.get(td.TAGS) if itype is None: return {'success': False, 'message': "Descriptor type is not supported by CRITs"} if not user.has_access_to(IndicatorACL.WRITE): return {'success': False, 'message': "User does not have permission to add Indicators to CRITs"} ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE)) results = handle_indicator_ind( obj.get(td.RAW_INDICATOR), "ThreatExchange", itype, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, request.user.username, method="ThreatExchange Service", reference="id: %s, owner: %s, share_level: %s" % (obj.get(td.ID), obj.get(td.OWNER)['name'], obj.get(td.SHARE_LEVEL)), add_domain=True, add_relationship=True, confidence=build_ci(obj.get(td.CONFIDENCE)), description=obj.get(td.DESCRIPTION), bucket_list=tags ) return results elif type_ == "Malware Analyses": if not user.has_access_to(SampleACL.WRITE): return {'success': False, 'message': "User does not have permission to add Sample to CRITs"} obj = Malware(id=id_) obj.details( fields=[f for f in Malware._fields if f not in (m.METADATA)] ) filename = obj.get(m.MD5) tags = obj.get(m.TAGS) try: data = obj.rf except: data = None results = handle_file( filename, data, "ThreatExchange", method="ThreatExchange Service", reference="id: %s, share_level: %s" % (obj.get(td.ID), obj.get(td.SHARE_LEVEL)), user=request.user.username, md5_digest = obj.get(m.MD5), sha1_digest = obj.get(m.SHA1), sha256_digest = obj.get(m.SHA256), size = obj.get(m.SAMPLE_SIZE), mimetype = obj.get(m.SAMPLE_TYPE), bucket_list=tags, ) return {'success': True, 'md5': results} else: return {'success': False, 'message': "Invalid Type"} return {'success': True}
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results if name == "URL" and "://" not in value.split('.')[0]: return {"success" : False, "message" : "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"} elif object_type == "Address": if "ipv4" in name: try: validate_ipv4_address(value) except DjangoValidationError: return {"success" : False, "message" : "Invalid IPv4 address. "} elif "ipv6" in name: try: validate_ipv6_address(value) except DjangoValidationError: return {"success" : False, "message" : "Invalid IPv6 address. "} elif "cidr" in name: try: if '/' not in value: raise ValidationError("") cidr_parts = value.split('/') if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128: raise ValidationError("") if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32: raise ValidationError("") validate_ipv46_address(cidr_parts[0]) except (ValidationError, ValueError) as cidr_error: return {"success" : False, "message" : "Invalid CIDR address. "} try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, related_id=oid, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) campaign = obj.campaign if hasattr(obj, 'campaign') else None ind_res = handle_indicator_ind(value, source, reference, object_type, analyst, method, add_domain=True, campaign=campaign, cache=cache) if ind_res['success']: ind = ind_res['object'] forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) else: results['message'] = "Object was added, but failed to add Indicator." \ "<br>Error: " + ind_res.get('message') if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships(analyst, meta=True) else: results['relationships'] = obj.sort_relationships(analyst, meta=True) else: results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() results['id'] = str(obj.id) return results except ValidationError, e: return {'success': False, 'message': str(e)}
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, indicator_campaign=None, indicator_campaign_confidence=None, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, parent_id=oid, parent_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) ind_res = handle_indicator_ind(value, source, reference, object_type, analyst, method=method, add_domain=True, campaign=indicator_campaign, campaign_confidence=indicator_campaign_confidence, cache=cache) if ind_res['success']: ind = ind_res['object'] # Inherit campaigns from top level item when creating # an indicator from an object if no campaigns were specified if indicator_campaign == None and ind != None: for campaign in obj.campaign: ec = EmbeddedCampaign(name=campaign.name, confidence=campaign.confidence, description="", analyst=analyst, date=datetime.datetime.now()) ind.add_campaign(ec) ind.save(username=analyst) forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships(analyst, meta=True) else: results['relationships'] = obj.sort_relationships(analyst, meta=True) else: results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() return results except ValidationError, e: return {'success': False, 'message': e}
def ip_add_update(ip_address, ip_type, source=None, source_method='', source_reference='', source_tlp=None, campaign=None, confidence='low', user=None, is_add_indicator=False, indicator_reference='', bucket_list=None, ticket=None, is_validate_only=False, cache={}, related_id=None, related_type=None, relationship_type=None, description=''): """ Add/update an IP address. :param ip_address: The IP to add/update. :type ip_address: str :param ip_type: The type of IP this is. :type ip_type: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param campaign: A campaign to attribute to this IP address. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param user: The user adding/updating this IP. :type user: str :param is_add_indicator: Also add an Indicator for this IP. :type is_add_indicator: bool :param indicator_reference: Reference for the indicator. :type indicator_reference: str :param bucket_list: Buckets to assign to this IP. :type bucket_list: str :param ticket: Ticket to assign to this IP. :type ticket: str :param is_validate_only: Only validate, do not add/update. :type is_validate_only: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_type: str :param relationship_type: Type of relationship to create. :type relationship_type: str :param description: A description for this IP :type description: str :returns: dict with keys: "success" (boolean), "message" (str), "object" (if successful) :class:`crits.ips.ip.IP` """ if not source: return {"success" : False, "message" : "Missing source information."} source_name = source (ip_address, error) = validate_and_normalize_ip(ip_address, ip_type) if error: return {"success": False, "message": error} retVal = {} is_item_new = False ip_object = None cached_results = cache.get(form_consts.IP.CACHED_RESULTS) if cached_results != None: ip_object = cached_results.get(ip_address) else: ip_object = IP.objects(ip=ip_address).first() if not ip_object: ip_object = IP() ip_object.ip = ip_address ip_object.ip_type = ip_type is_item_new = True if cached_results != None: cached_results[ip_address] = ip_object if not ip_object.description: ip_object.description = description or '' elif ip_object.description != description: ip_object.description += "\n" + (description or '') if isinstance(source_name, basestring): if user.check_source_write(source): source = [create_embedded_source(source, reference=source_reference, method=source_method, tlp=source_tlp, analyst=user.username)] else: return {"success":False, "message": "User does not have permission to add object \ using source %s." % source} if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user.username) campaign = [c] if campaign: for camp in campaign: ip_object.add_campaign(camp) if source: for s in source: ip_object.add_source(s) else: return {"success" : False, "message" : "Missing source information."} if bucket_list: ip_object.add_bucket_list(bucket_list, user.username) if ticket: ip_object.add_ticket(ticket, user.username) related_obj = None if related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip]) if is_validate_only == False: ip_object.save(username=user.username) #set the URL for viewing the new data if is_item_new == True: retVal['message'] = ('Success! Click here to view the new IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) else: message = ('Updated existing IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message elif is_validate_only == True: if ip_object.id != None and is_item_new == False: message = ('Warning: IP already exists: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message if is_add_indicator: from crits.indicators.handlers import handle_indicator_ind result = handle_indicator_ind(ip_address, source_name, ip_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, source_method=source_method, source_reference = indicator_reference, source_tlp = source_tlp, add_domain=False, add_relationship=True, bucket_list=bucket_list, ticket=ticket, cache=cache) if related_obj and ip_object and relationship_type: relationship_type=RelationshipTypes.inverse(relationship=relationship_type) ip_object.add_relationship(related_obj, relationship_type, analyst=user.username, get_rels=False) ip_object.save(username=user.username) # run ip triage if is_item_new and is_validate_only == False: ip_object.reload() run_triage(ip_object, user) retVal['success'] = True retVal['object'] = ip_object return retVal
def parse_cybox_object(self, cbx_obj, description='', ind_id=None): """ Parse a CybOX object form a STIX doc. An object can contain multiple related_objects, which in turn can have their own related_objects, so this handles those recursively. :param cbx_obj: The CybOX object to parse. :type cbx_obj: A CybOX object. :param description: Parent-level (e.g. Observable) description. :type description: str :param ind_id: The ID of a parent STIX Indicator. :type ind_id: str """ # check for missing attributes if not cbx_obj or not cbx_obj.properties: if cbx_obj.idref: # just a reference, so nothing to parse return else: cbx_id = getattr(cbx_obj, 'id_', 'None') self.failed.append(("No valid object_properties was found!", "Observable (%s)" % cbx_id, cbx_id)) # note for display in UI return # Don't parse if already been parsed # This is for artifacts that are related to CybOX File Objects if cbx_obj.id_ in self.parsed: return try: # try to create CRITs object from Cybox Object analyst = self.source_instance.analyst item = cbx_obj.properties val = cbx_obj.id_ if isinstance(item, Address) and not ind_id: if item.category in ('cidr', 'ipv4-addr', 'ipv4-net', 'ipv4-netmask', 'ipv6-addr', 'ipv6-net', 'ipv6-netmask'): imp_type = "IP" for value in item.address_value.values: val = str(value).strip() if self.preview: res = None else: iptype = get_crits_ip_type(item.category) if iptype: res = ip_add_update(val, iptype, [self.source], analyst=analyst, is_add_indicator=True) else: res = {'success': False, 'reason': 'No IP Type'} self.parse_res(imp_type, val, cbx_obj, res, ind_id) if (not ind_id and (isinstance(item, DomainName) or (isinstance(item, URI) and item.type_ == 'Domain Name'))): imp_type = "Domain" for val in item.value.values: if self.preview: res = None else: res = upsert_domain(str(val), [self.source], username=analyst) self.parse_res(imp_type, str(val), cbx_obj, res, ind_id) elif isinstance(item, HTTPSession): imp_type = "RawData" val = cbx_obj.id_ try: c_req = item.http_request_response[0].http_client_request hdr = c_req.http_request_header if hdr.raw_header: data = hdr.raw_header.value title = "HTTP Header from STIX: %s" % self.package.id_ method = self.source_instance.method ref = self.source_instance.reference if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="HTTP Header", tool_name="STIX", tool_version=None, method=method, reference=ref) else: imp_type = "Indicator" ind_type = "HTTP Request Header Fields - User-Agent" val = hdr.parsed_header.user_agent.value val = ','.join(val) if isinstance(val, list) else val if self.preview: res = None else: res = handle_indicator_ind(val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_relationship=True, description=description) except: msg = "Unsupported use of 'HTTPSession' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, WhoisEntry): # No sure where else to put this imp_type = "RawData" val = cbx_obj.id_ if item.remarks: data = item.remarks.value title = "WHOIS Entry from STIX: %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file(data, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="WHOIS", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) else: msg = "Unsupported use of 'WhoisEntry' object." res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, Artifact): # Not sure if this is right, and I believe these can be # encoded in a couple different ways. imp_type = "RawData" val = cbx_obj.id_ rawdata = item.data.decode('utf-8') # TODO: find out proper ways to determine title, datatype, # tool_name, tool_version title = "Artifact for Event: STIX Document %s" % self.package.id_ if self.preview: res = None val = title else: res = handle_raw_data_file(rawdata, self.source.name, user=analyst, description=description, title=title, data_type="Text", tool_name="STIX", tool_version=None, method=self.source_instance.method, reference=self.source_instance.reference) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif (isinstance(item, File) and item.custom_properties and item.custom_properties[0].name == "crits_type" and item.custom_properties[0]._value == "Certificate"): imp_type = "Certificate" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if isinstance(rel_obj.properties, Artifact): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_cert_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File) and self.has_network_artifact(item): imp_type = "PCAP" val = str(item.file_name) data = None if self.preview: res = None else: for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_NETWORK): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) res = handle_pcap_file(val, data, self.source, user=analyst, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, File): imp_type = "Sample" md5 = item.md5 if md5: md5 = md5.lower() val = str(item.file_name or md5) # add sha1/sha256/ssdeep once handle_file supports it size = item.size_in_bytes data = None if item.file_path: path = "File Path: " + str(item.file_path) description += "\n" + path for rel_obj in item.parent.related_objects: if (isinstance(rel_obj.properties, Artifact) and rel_obj.properties.type_ == Artifact.TYPE_FILE): data = rel_obj.properties.data self.parsed.append(rel_obj.id_) if not md5 and not data and val and val != "None": imp_type = "Indicator" if self.preview: res = None else: res = handle_indicator_ind(val, self.source, "Win File", IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) elif md5 or data: if self.preview: res = None else: res = handle_file(val, data, self.source, user=analyst, md5_digest=md5, is_return_only_md5=False, size=size, description=description) else: val = cbx_obj.id_ msg = "CybOX 'File' object has no MD5, data, or filename" res = {'success': False, 'reason': msg} self.parse_res(imp_type, val, cbx_obj, res, ind_id) elif isinstance(item, EmailMessage): imp_type = 'Email' id_list = [] data = {} val = cbx_obj.id_ get_attach = False data['raw_body'] = str(item.raw_body) data['raw_header'] = str(item.raw_header) data['helo'] = str(item.email_server) if item.header: data['subject'] = str(item.header.subject) if item.header.date: data['date'] = item.header.date.value val = "Date: %s, Subject: %s" % (data.get('date', 'None'), data['subject']) data['message_id'] = str(item.header.message_id) data['sender'] = str(item.header.sender) data['reply_to'] = str(item.header.reply_to) data['x_originating_ip'] = str(item.header.x_originating_ip) data['x_mailer'] = str(item.header.x_mailer) data['boundary'] = str(item.header.boundary) data['from_address'] = str(item.header.from_) if item.header.to: data['to'] = [str(r) for r in item.header.to.to_list()] if data.get('date'): # Email TLOs must have a date data['source'] = self.source.name data['source_method'] = self.source_instance.method data['source_reference'] = self.source_instance.reference if self.preview: res = None else: res = handle_email_fields(data, analyst, "STIX") self.parse_res(imp_type, val, cbx_obj, res, ind_id) if not self.preview and res.get('status'): id_list.append(cbx_obj.id_) # save ID for atchmnt rels get_attach = True else: # Can't be an Email TLO, so save fields for x, key in enumerate(data): if data[key] and data[key] != "None": if key in ('raw_header', 'raw_body'): if key == 'raw_header': title = "Email Header from STIX Email: %s" d_type = "Email Header" else: title = "Email Body from STIX Email: %s" d_type = "Email Body" imp_type = 'RawData' title = title % cbx_obj.id_ if self.preview: res = None else: res = handle_raw_data_file(data[key], self.source, analyst, description, title, d_type, "STIX", self.stix_version) self.parse_res(imp_type, title, cbx_obj, res, ind_id) elif key == 'to': imp_type = 'Target' for y, addr in enumerate(data[key]): tgt_dict = {'email_address': addr} if self.preview: res = None else: res = upsert_target(tgt_dict, analyst) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x, y) self.parse_res(imp_type, addr, tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) else: imp_type = 'Indicator' if key in ('sender', 'reply_to', 'from_address'): ind_type = "Address - e-mail" elif 'ip' in key: ind_type = "Address - ipv4-addr" elif key == 'raw_body': ind_type = "Email Message" else: ind_type = "String" if self.preview: res = None else: res = handle_indicator_ind(data[key], self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) if res['success']: get_attach = True tmp_obj = copy(cbx_obj) tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x) self.parse_res(imp_type, data[key], tmp_obj, res, ind_id) self.ind2obj.setdefault(cbx_obj.id_, []).append(tmp_obj.id_) id_list.append(tmp_obj.id_) if not self.preview: # Setup relationships between all Email attributes for oid in id_list: for oid2 in id_list: if oid != oid2: self.relationships.append((oid, RelationshipTypes.RELATED_TO, oid2, "High")) # Should check for attachments and add them here. if get_attach and item.attachments: for attach in item.attachments: rel_id = attach.to_dict()['object_reference'] for oid in id_list: self.relationships.append((oid, RelationshipTypes.CONTAINS, rel_id, "High")) else: # try to parse all other possibilities as Indicator imp_type = "Indicator" val = cbx_obj.id_ c_obj = make_crits_object(item) # Ignore what was already caught above if (ind_id or c_obj.object_type not in IPTypes.values()): ind_type = c_obj.object_type for val in [str(v).strip() for v in c_obj.value if v]: if ind_type: # handle domains mislabeled as URLs if c_obj.object_type == 'URI' and '/' not in val: ind_type = "Domain" if self.preview: res = None else: res = handle_indicator_ind(val, self.source, ind_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, analyst, add_domain=True, add_relationship=True, description=description) self.parse_res(imp_type, val, cbx_obj, res, ind_id) except Exception, e: # probably caused by cybox object we don't handle self.failed.append((e.message, "%s (%s)" % (imp_type, val), cbx_obj.id_)) # note for display in UI