def add_campaign_from_nodes(name, confidence, nodes, user): result = { "success": False } # Make sure Campaign exists campaign_obj = Campaign.objects(name=name).first() if not campaign_obj: result["message"] = "Invalid campaign name." return result campaign = EmbeddedCampaign(name=name, confidence=confidence, analyst=user) counter = 0 for node in nodes: id_ = node.get('id', None) type_ = node.get('type', None) # Must have type and id, and type must not be Campaign if not id_ or not type_ or type_.lower() == 'campaign': continue obj = class_from_id(type_, id_) if not obj: continue obj.add_campaign(campaign) obj.save() counter += 1 result["message"] = "%s nodes processed" % counter result["success"] = True return result
def campaign_add(campaign_name, confidence, description, related, analyst, ctype=None, oid=None, obj=None, update=True): """ Attribute a Campaign to a top-level object. :param campaign_name: The Campaign to attribute. :type campaign_name: str :param confidence: The confidence level of this attribution (low, medium, high) :type confidence: str :param description: Description of this attribution. :type description: str :param related: Should this attribution propagate to related top-level objects. :type related: boolean :param analyst: The user attributing this Campaign. :type analyst: str :param ctype: The top-level object type. :type ctype: str :param oid: The ObjectId of the top-level object. :type oid: str :param obj: The top-level object instantiated class. :type obj: Instantiated class object :param update: If True, allow merge with pre-existing campaigns : If False, do not change any pre-existing campaigns :type update: boolean :returns: dict with keys: 'success' (boolean), 'html' (str) if successful, 'message' (str). """ if not obj: if ctype and oid: # Verify the document exists. obj = class_from_id(ctype, oid) if not obj: return {'success': False, 'message': 'Cannot find %s.' % ctype} else: return {'success': False, 'message': 'Object type and ID, or object instance, must be provided.'} # Create the embedded campaign. campaign = EmbeddedCampaign(name=campaign_name, confidence=confidence, description=description, analyst=analyst) result = obj.add_campaign(campaign, update=update) if result['success']: if related: campaign_addto_related(obj, campaign, analyst) try: obj.save(username=analyst) html = obj.format_campaign(campaign, analyst) return {'success': True, 'html': html, 'message': result['message']} except ValidationError, e: return {'success': False, 'message': "Invalid value: %s" % e}
def campaign_edit(ctype, oid, campaign_name, confidence, description, date, related, analyst): """ Edit an attributed Campaign for a top-level object. :param ctype: The top-level object type. :type ctype: str :param oid: The ObjectId of the top-level object. :type oid: str :param campaign_name: The Campaign to attribute. :type campaign_name: str :param confidence: The confidence level of this attribution (low, medium, high) :type confidence: str :param description: Description of this attribution. :type description: str :param date: The date of attribution. :type date: :class:`datetime.datetime` :param related: Should this attribution propagate to related top-level objects. :type related: boolean :param analyst: The user editing this attribution. :type analyst: str :returns: dict with keys: 'success' (boolean), 'html' (str) if successful, 'message' (str) if failed. """ # Verify the document exists. crits_object = class_from_id(ctype, oid) if not crits_object: return {'success': False, 'message': 'Cannot find %s.' % ctype} # Create the embedded campaign. campaign = EmbeddedCampaign(name=campaign_name, confidence=confidence, description=description, analyst=analyst, date=date) crits_object.edit_campaign(campaign_item=campaign) if related: campaign_addto_related(crits_object, campaign, analyst) try: crits_object.save(username=analyst) html = crits_object.format_campaign(campaign, analyst) return {'success': True, 'html': html} except ValidationError, e: return {'success': False, 'message': "Invalid value: %s" % e}
def upload_file(request, related_md5=None): """ Upload a new sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param related_md5: The MD5 of a related sample. :type related_md5: str :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': form = UploadFileForm(request.user, request.POST, request.FILES) email_errmsg = None if form.is_valid(): response = { 'success': False, 'message': 'Unknown error; unable to upload file.' } inherited_source = None backdoor = form.cleaned_data['backdoor'] campaign = form.cleaned_data['campaign'] confidence = form.cleaned_data['confidence'] source = form.cleaned_data['source_name'] source_method = form.cleaned_data['source_method'] source_reference = form.cleaned_data['source_reference'] source_tlp = form.cleaned_data['source_tlp'] user = request.user description = form.cleaned_data['description'] related_id = form.cleaned_data.get('related_id', None) related_type = form.cleaned_data.get('related_type', None) relationship_type = form.cleaned_data.get('relationship_type', None) if related_md5: reload_page = True else: reload_page = False related_md5 = form.cleaned_data['related_md5'] if related_md5: related_sample = Sample.objects(md5=related_md5).first() if not related_sample: response['message'] = ( "Upload Failed. Unable to locate related sample. %s" % related_md5) return render(request, "file_upload_response.html", {'response': json.dumps(response)}) # If selected, new sample inherits the campaigns of the related sample. if form.cleaned_data['inherit_campaigns']: if campaign: related_sample.campaign.append( EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user)) campaign = related_sample.campaign # If selected, new sample inherits the sources of the related sample if form.cleaned_data['inherit_sources']: inherited_source = related_sample.source elif related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: response['success'] = False response['message'] = ( "Upload Failed. Unable to locate related Item") return render( request, "file_upload_response.html", {'response': json.dumps(response)}, ) else: if form.cleaned_data['inherit_campaigns']: if campaign: related_obj.campaign.append( EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user)) campaign = related_obj.campaign if form.cleaned_data['inherit_sources']: inherited_source = related_obj.source backdoor_name = None backdoor_version = None if backdoor: backdoor = backdoor.split('|||') if len(backdoor) == 2: (backdoor_name, backdoor_version) = backdoor[0], backdoor[1] try: if request.FILES: result = handle_uploaded_file( request.FILES['filedata'], source, source_method=source_method, source_reference=source_reference, source_tlp=source_tlp, file_format=form.cleaned_data['file_format'], password=form.cleaned_data['password'], user=user, campaign=campaign, confidence=confidence, related_md5=related_md5, related_id=related_id, related_type=related_type, relationship_type=relationship_type, bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, backdoor_name=backdoor_name, backdoor_version=backdoor_version, description=description) else: result = handle_uploaded_file( None, source, source_method=source_method, source_reference=source_reference, source_tlp=source_tlp, file_format=form.cleaned_data['file_format'], password=None, user=user, campaign=campaign, confidence=confidence, related_md5=related_md5, related_id=related_id, related_type=related_type, relationship_type=relationship_type, filename=request.POST['filename'].strip(), md5=request.POST['md5'].strip().lower(), sha1=request.POST['sha1'].strip().lower(), sha256=request.POST['sha256'].strip().lower(), bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, is_return_only_md5=False, backdoor_name=backdoor_name, backdoor_version=backdoor_version, description=description) except ZipFileError, zfe: return render( request, 'file_upload_response.html', { 'response': json.dumps({ 'success': False, 'message': zfe.value }) }) else: # zip file upload, etc; result is a list of strings (1 hash per file) if len(result) > 0 and not isinstance(result[0], dict): filedata = request.FILES['filedata'] message = ('<a href="%s">View Uploaded Samples.</a>' % reverse('crits-samples-views-view_upload_list', args=[filedata.name, result])) response = {'success': True, 'message': message} md5_response = result # regular file upload; result is a list with a single dict else: response['success'] = result[0].get('success', False) response['message'] = result[0].get( 'message', response.get('message')) try: md5_response = [result[0].get('object').md5] except: md5_response = None if response['success']: if request.POST.get('email') and md5_response: for s in md5_response: email_errmsg = mail_sample(s, [request.user.email]) if email_errmsg is not None: msg = "<br>Error emailing sample %s: %s\n" % ( s, email_errmsg) response['message'] = response['message'] + msg if reload_page: response['redirect_url'] = reverse( 'crits-samples-views-detail', args=[related_md5]) return render(request, "file_upload_response.html", {'response': json.dumps(response)}) else: if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear form.fields['related_md5'].widget = forms.HiddenInput() return render( request, 'file_upload_response.html', { 'response': json.dumps({ 'success': False, 'form': form.as_table() }) })
def upload_file(request, related_md5=None): """ Upload a new sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param related_md5: The MD5 of a related sample. :type related_md5: str :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': form = UploadFileForm(request.user, request.POST, request.FILES) email_errmsg = None if form.is_valid(): response = { 'success': False, 'message': 'Unknown error; unable to upload file.' } inherited_source = None backdoor = form.cleaned_data['backdoor'] campaign = form.cleaned_data['campaign'] confidence = form.cleaned_data['confidence'] source = form.cleaned_data['source'] method = form.cleaned_data['method'] reference = form.cleaned_data['reference'] analyst = request.user.username if related_md5: reload_page = True else: reload_page = False related_md5 = form.cleaned_data['related_md5'] if related_md5: related_sample = Sample.objects(md5=related_md5).first() if not related_sample: response[ 'message'] = "Upload Failed. Unable to locate related sample." return render_to_response( "file_upload_response.html", {'response': json.dumps(response)}, RequestContext(request)) # If selected, new sample inherits the campaigns of the related sample. if form.cleaned_data['inherit_campaigns']: if campaign: related_sample.campaign.append( EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)) campaign = related_sample.campaign # If selected, new sample inherits the sources of the related sample if form.cleaned_data['inherit_sources']: inherited_source = related_sample.source backdoor_name = None backdoor_version = None if backdoor: backdoor = backdoor.split('|||') if len(backdoor) == 2: (backdoor_name, backdoor_version) = backdoor[0], backdoor[1] try: if request.FILES: result = handle_uploaded_file( request.FILES['filedata'], source, method=method, reference=reference, file_format=form.cleaned_data['file_format'], password=form.cleaned_data['password'], user=analyst, campaign=campaign, confidence=confidence, related_md5=related_md5, bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, backdoor_name=backdoor_name, backdoor_version=backdoor_version) else: result = handle_uploaded_file( None, source, method=method, reference=reference, file_format=form.cleaned_data['file_format'], password=None, user=analyst, campaign=campaign, confidence=confidence, related_md5=related_md5, filename=request.POST['filename'].strip(), md5=request.POST['md5'].strip().lower(), sha1=request.POST['sha1'].strip().lower(), sha256=request.POST['sha256'].strip().lower(), bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], inherited_source=inherited_source, is_return_only_md5=False, backdoor_name=backdoor_name, backdoor_version=backdoor_version) except ZipFileError, zfe: return render_to_response( 'file_upload_response.html', { 'response': json.dumps({ 'success': False, 'message': zfe.value }) }, RequestContext(request)) else: if len(result) > 1: filedata = request.FILES['filedata'] message = ('<a href="%s">View Uploaded Samples.</a>' % reverse('crits.samples.views.view_upload_list', args=[filedata.name, result])) response = {'success': True, 'message': message} md5_response = result elif len(result) == 1: md5_response = None if not request.FILES: response['success'] = result[0].get('success', False) if (response['success'] == False): response['message'] = result[0].get( 'message', response.get('message')) else: md5_response = [result[0].get('object').md5] else: md5_response = [result[0]] response['success'] = True if md5_response != None: response['message'] = ( 'File uploaded successfully. <a href="%s">View Sample.</a>' % reverse('crits.samples.views.detail', args=md5_response)) if response['success']: if request.POST.get('email'): for s in md5_response: email_errmsg = mail_sample(s, [request.user.email]) if email_errmsg is not None: msg = "<br>Error emailing sample %s: %s\n" % ( s, email_errmsg) response['message'] = response['message'] + msg if reload_page: response['redirect_url'] = reverse( 'crits.samples.views.detail', args=[related_md5]) return render_to_response("file_upload_response.html", {'response': json.dumps(response)}, RequestContext(request)) else: if related_md5: #if this is a 'related' upload, hide field so it doesn't reappear form.fields['related_md5'].widget = forms.HiddenInput() return render_to_response( 'file_upload_response.html', { 'response': json.dumps({ 'success': False, 'form': form.as_table() }) }, RequestContext(request))
def add_new_backdoor(name, version=None, aliases=None, description=None, source=None, source_method=None, source_reference=None, campaign=None, confidence=None, user=None, bucket_list=None, ticket=None): """ Add an Backdoor to CRITs. :param name: The name of the backdoor. :type name: str :param version: Version of the backdoor. :type version: str :param aliases: Aliases for the backdoor. :type aliases: list or str :param description: Description of the backdoor. :type description: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param campaign: A campaign to attribute to this backdoor. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param user: The user adding this backdoor. :type user: str :param bucket_list: Buckets to assign to this backdoor. :type bucket_list: str :param ticket: Ticket to assign to this backdoor. :type ticket: str :returns: dict with keys: "success" (boolean), "message" (str), "id" (str), "object" (if successful) :class:`crits.backdoors.backdoor.Backdoor` """ retVal = {'success': False, 'message': ''} if isinstance(source, basestring): source = [create_embedded_source(source, reference=source_reference, method=source_method, analyst=user)] elif isinstance(source, EmbeddedSource): source = [source] if not source: retVal['message'] = "Missing source information." return retVal # When creating a backdoor object we can potentially create multiple # objects. If we are given a name but no version we will create an object # with just the name (called the "family backdoor"). If given a name and a # version we will create the family backdoor and the specific backdoor for # that given version. # In case we create more than one backdoor object, store the created ones # in this list. The list is walked later on and attributes applied to each # object. objs = [] # First check if we have the family (name and no version). family = Backdoor.objects(name=name, version='').first() if not family: # Family does not exist, new object. Details are handled later. family = Backdoor() family.name = name family.version = '' objs.append(family) # Now check if we have the specific instance for this name + version. backdoor = None if version: backdoor = Backdoor.objects(name=name, version=version).first() if not backdoor: # Backdoor does not exist, new object. Details are handled later. backdoor = Backdoor() backdoor.name = name backdoor.version = version objs.append(backdoor) # At this point we have a family object and potentially a specific object. # Add the common parameters to all objects in the list and save them. for backdoor in objs: for s in source: backdoor.add_source(s) # Don't overwrite existing description. if description and backdoor.description == '': backdoor.description = description.strip() if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user) campaign = [c] if campaign: for camp in campaign: backdoor.add_campaign(camp) if aliases: if isinstance(aliases, basestring): aliases = aliases.split(',') for alias in aliases: alias = alias.strip() if alias not in backdoor.aliases: backdoor.aliases.append(alias) if bucket_list: backdoor.add_bucket_list(bucket_list, user) if ticket: backdoor.add_ticket(ticket, user) backdoor.save(username=user) # run backdoor triage backdoor.reload() run_triage(backdoor, user) # Because family objects are put in the list first we will always # return a link to the most specific object created. If there is only # one item in the list it will be the family object. resp_url = reverse('crits.backdoors.views.backdoor_detail', args=[backdoor.id]) retVal['message'] = 'Success: <a href="%s">%s</a>' % (resp_url, backdoor.name) retVal['object'] = backdoor retVal['id'] = str(backdoor.id) # If we have a family and specific object, attempt to relate the two. if len(objs) == 2: objs[0].add_relationship(objs[1], RelationshipTypes.RELATED_TO) objs[0].save() retVal['success'] = True return retVal
def add_new_actor(name, aliases=None, description=None, source=None, source_method='', source_reference='', campaign=None, confidence=None, analyst=None, bucket_list=None, ticket=None): """ Add an Actor to CRITs. :param name: The name of the Actor. :type name: str :param aliases: Aliases for the actor. :type aliases: list or str :param description: Description of the actor. :type description: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param campaign: A campaign to attribute to this actor. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param analyst: The user adding this actor. :type analyst: str :param bucket_list: Buckets to assign to this actor. :type bucket_list: str :param ticket: Ticket to assign to this actor. :type ticket: str :returns: dict with keys: "success" (boolean), "message" (str), "object" (if successful) :class:`crits.actors.actor.Actor` """ is_item_new = False retVal = {} actor = Actor.objects(name=name).first() if not actor: actor = Actor() actor.name = name if description: actor.description = description.strip() is_item_new = True if isinstance(source, basestring): source = [ create_embedded_source(source, reference=source_reference, method=source_method, analyst=analyst) ] if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst) campaign = [c] if campaign: for camp in campaign: actor.add_campaign(camp) if source: for s in source: actor.add_source(s) else: return {"success": False, "message": "Missing source information."} if not isinstance(aliases, list): aliases = aliases.split(',') for alias in aliases: alias = alias.strip() if alias not in actor.aliases: actor.aliases.append(alias) if bucket_list: actor.add_bucket_list(bucket_list, analyst) if ticket: actor.add_ticket(ticket, analyst) actor.save(username=analyst) # run actor triage if is_item_new: actor.reload() run_triage(actor, analyst) resp_url = reverse('crits.actors.views.actor_detail', args=[actor.id]) retVal['message'] = ('Success! Click here to view the new Actor: ' '<a href="%s">%s</a>' % (resp_url, actor.name)) retVal['success'] = True retVal['object'] = actor retVal['id'] = str(actor.id) return retVal
def add_new_actor(name, aliases=None, description=None, source=None, source_method='', source_reference='', source_tlp=None, campaign=None, confidence=None, user=None, bucket_list=None, ticket=None, related_id=None, related_type=None, relationship_type=None): """ Add an Actor to CRITs. :param name: The name of the Actor. :type name: str :param aliases: Aliases for the actor. :type aliases: list or str :param description: Description of the actor. :type description: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param source_tlp: The TLP for this Actor. :type source_tlp: str :param campaign: A campaign to attribute to this actor. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param user: The user adding this actor. :type user: :class:`crits.core.user.CRITsUser` :param bucket_list: Buckets to assign to this actor. :type bucket_list: str :param ticket: Ticket to assign to this actor. :type ticket: str :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_id: str :param relationship_type: Type of relationship to create. :type relationship_type: str :returns: dict with keys: "success" (boolean), "message" (str), "object" (if successful) :class:`crits.actors.actor.Actor` """ username = user.username is_item_new = False retVal = {} actor = Actor.objects(name=name).first() if not actor: actor = Actor() actor.name = name if description: actor.description = description.strip() is_item_new = True if isinstance(source, basestring): if user.check_source_write(source): source = [create_embedded_source(source, reference=source_reference, method=source_method, tlp=source_tlp, analyst=username)] else: return {"success": False, "message": "User does not have permission to add objects \ using source %s." % str(source)} if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username) campaign = [c] if campaign: for camp in campaign: actor.add_campaign(camp) if source: for s in source: actor.add_source(s) else: return {"success" : False, "message" : "Missing source information."} if not isinstance(aliases, list): aliases = aliases.split(',') for alias in aliases: alias = alias.strip() if alias not in actor.aliases: actor.aliases.append(alias) if bucket_list: actor.add_bucket_list(bucket_list, username) if ticket: actor.add_ticket(ticket, username) related_obj = None if related_id and related_type: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal actor.save(username=username) if related_obj and actor: relationship_type=RelationshipTypes.inverse(relationship=relationship_type) actor.add_relationship(related_obj, relationship_type, analyst=username, get_rels=False) actor.save(username=username) actor.reload() # run actor triage if is_item_new: actor.reload() run_triage(actor, user) resp_url = reverse('crits.actors.views.actor_detail', args=[actor.id]) retVal['message'] = ('Success! Click here to view the new Actor: ' '<a href="%s">%s</a>' % (resp_url, actor.name)) retVal['success'] = True retVal['object'] = actor retVal['id'] = str(actor.id) return retVal
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}): """ Add a new domain to CRITs. :param data: The data about the domain. :type data: dict :param request: The Django request. :type request: :class:`django.http.HttpRequest` :param errors: A list of current errors to append to. :type errors: list :param rowData: Any objects that need to be added to the domain. :type rowData: dict :param is_validate_only: Only validate the data and return any errors. :type is_validate_only: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: tuple """ username = request.user.username result = False retVal = {} reference = data.get('domain_reference') name = data.get('domain_source') method = data.get('domain_method') source = [ create_embedded_source(name, reference=reference, method=method, analyst=username) ] bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if data.get('campaign') and data.get('confidence'): campaign = [ EmbeddedCampaign(name=data.get('campaign'), confidence=data.get('confidence'), analyst=username) ] else: campaign = [] (sdomain, fqdn) = get_domain(data['domain']) if sdomain == "no_tld_found_error": errors.append(u"Error: Invalid domain: " + data['domain']) elif is_validate_only == False: retVal = upsert_domain(sdomain, fqdn, source, username, campaign, bucket_list=bucket_list, ticket=ticket, cache=cache) ip_result = None if retVal['success']: new_domain = retVal['object'] add_ip = data.get('add_ip') if add_ip: ip = data.get('ip') ip_type = data.get('ip_type') if data.get('same_source'): ip_source = data.get('domain_source') ip_method = data.get('domain_method') ip_reference = reference else: ip_source = data.get('ip_source') ip_method = data.get('ip_method') ip_reference = data.get('ip_reference') from crits.ips.handlers import ip_add_update ip_result = ip_add_update(ip, ip_type, ip_source, ip_method, ip_reference, campaign=campaign, analyst=username, bucket_list=bucket_list, ticket=ticket, cache=cache) if not ip_result['success']: errors.append(ip_result['message']) else: #add a relationship with the new IP address new_ip = ip_result['object'] if new_domain and new_ip: new_domain.add_relationship(rel_item=new_ip, rel_type='Resolved_To', analyst=username, get_rels=False) new_domain.save(username=username) new_ip.save(username=username) #set the URL for viewing the new data resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn]) if retVal['is_domain_new'] == True: retVal['message'] = ( 'Success! Click here to view the new domain: ' '<a href="%s">%s</a>' % (resp_url, fqdn)) else: message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, fqdn)) retVal['message'] = message retVal[form_consts.Status. STATUS_FIELD] = form_consts.Status.DUPLICATE retVal['warning'] = message #add indicators if data.get('add_indicators'): from crits.indicators.handlers import create_indicator_from_obj # If we have an IP object, add an indicator for that. if ip_result and ip_result['success']: obj = ip_result['object'] result = create_indicator_from_obj('Address - ipv4-addr', 'IP', obj.id, obj.ip, username) if result['success'] == False: errors.append(result['message']) # Add an indicator for the domain. result = create_indicator_from_obj('URI - Domain Name', 'Domain', new_domain.id, sdomain, username) if result['success'] == False: errors.append(result['message']) # If we have an FQDN (ie: it is not the same as sdomain) # then add that also. if fqdn != sdomain: result = create_indicator_from_obj('URI - Domain Name', 'Domain', new_domain.id, fqdn, username) if result['success'] == False: errors.append(result['message']) result = True elif 'message' in retVal: #database error? (!c_dom) errors.append( retVal['message']) #u"Unknown error: unable to add domain") elif is_validate_only == True: domain = data['domain'] fqdn_domain = retrieve_domain(domain, cache) if fqdn_domain: if isinstance(fqdn_domain, Domain): resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn]) message = ('Warning: Domain already exists: ' '<a href="%s">%s</a>' % (resp_url, fqdn)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message else: result_cache = cache.get(form_consts.Domain.CACHED_RESULTS) result_cache[domain.lower()] = True # This block tries to add objects to the item if retVal.get('success') or is_validate_only == True: if rowData: objectsData = rowData.get(form_consts.Common.OBJECTS_DATA) # add new objects if they exist if objectsData: objectsData = json.loads(objectsData) current_domain = retrieve_domain(fqdn, cache) for object_row_counter, objectData in enumerate( objectsData, 1): if current_domain != None: # if the domain exists then try to add objects to it if isinstance(current_domain, Domain) == True: objectDict = object_array_to_dict( objectData, "Domain", current_domain.id) else: objectDict = object_array_to_dict( objectData, "Domain", "") current_domain = None else: objectDict = object_array_to_dict( objectData, "Domain", "") (object_result, object_errors, object_retVal) = validate_and_add_new_handler_object( None, objectDict, request, errors, object_row_counter, is_validate_only=is_validate_only, cache=cache, obj=current_domain) if object_retVal.get('success') == False: retVal['success'] = False if object_retVal.get('message'): errors.append(object_retVal['message']) return result, errors, retVal
def upsert_target(data, analyst): """ Add/update target information. :param data: The target information. :type data: dict :param analyst: The user adding the target. :type analyst: str :returns: dict with keys "success" (boolean) and "message" (str) """ if 'email_address' not in data: return {'success': False, 'message': "No email address to look up"} # check for exact match first target = Target.objects(email_address=data['email_address']).first() if not target: # if no exact match, look for case-insensitive match target = Target.objects( email_address__iexact=data['email_address']).first() is_new = False if not target: is_new = True target = Target() target.email_address = data['email_address'].strip().lower() bucket_list = False ticket = False if 'department' in data: target.department = data['department'] if 'division' in data: target.division = data['division'] if 'organization_id' in data: target.organization_id = data['organization_id'] if 'firstname' in data: target.firstname = data['firstname'] if 'lastname' in data: target.lastname = data['lastname'] if 'note' in data: target.note = data['note'] if 'title' in data: target.title = data['title'] if 'campaign' in data and 'camp_conf' in data: target.add_campaign( EmbeddedCampaign(name=data['campaign'], confidence=data['camp_conf'], analyst=analyst)) if 'bucket_list' in data: bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) if 'ticket' in data: ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if bucket_list: target.add_bucket_list(bucket_list, analyst) if ticket: target.add_ticket(ticket, analyst) try: target.save(username=analyst) target.reload() if is_new: run_triage(target, analyst) return { 'success': True, 'message': "Target saved successfully", 'id': str(target.id) } except ValidationError, e: return {'success': False, 'message': "Target save failed: %s" % e}
def ip_add_update(ip_address, ip_type, source=None, source_method=None, source_reference=None, campaign=None, confidence='low', analyst=None, is_add_indicator=False, indicator_reference=None, bucket_list=None, ticket=None, is_validate_only=False, cache={}): """ Add/update an IP address. :param ip_address: The IP to add/update. :type ip_address: str :param ip_type: The type of IP this is. :type ip_type: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param campaign: A campaign to attribute to this IP address. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param analyst: The user adding/updating this IP. :type analyst: str :param is_add_indicator: Also add an Indicator for this IP. :type is_add_indicator: bool :param indicator_reference: Reference for the indicator. :type indicator_reference: str :param bucket_list: Buckets to assign to this IP. :type bucket_list: str :param ticket: Ticket to assign to this IP. :type ticket: str :param is_validate_only: Only validate, do not add/update. :type is_validate_only: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str), "object" (if successful) :class:`crits.ips.ip.IP` """ retVal = {} is_item_new = False ip_object = None cached_results = cache.get(form_consts.IP.CACHED_RESULTS) if cached_results != None: ip_object = cached_results.get(ip_address) else: ip_object = IP.objects(ip=ip_address).first() if not ip_object: ip_object = IP() ip_object.ip = ip_address ip_object.ip_type = ip_type is_item_new = True if cached_results != None: cached_results[ip_address] = ip_object if isinstance(source, basestring): source = [create_embedded_source(source, reference=source_reference, method=source_method, analyst=analyst)] if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst) campaign = [c] if campaign: for camp in campaign: ip_object.add_campaign(camp) if source: for s in source: ip_object.add_source(s) if bucket_list: ip_object.add_bucket_list(bucket_list, analyst) if ticket: ip_object.add_ticket(ticket, analyst) resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip]) if is_validate_only == False: ip_object.save(username=analyst) #set the URL for viewing the new data if is_item_new == True: retVal['message'] = ('Success! Click here to view the new IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) else: message = ('Updated existing IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message elif is_validate_only == True: if ip_object.id != None and is_item_new == False: message = ('Warning: IP already exists: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message if is_add_indicator: from crits.indicators.handlers import handle_indicator_ind handle_indicator_ind(ip_address, source, indicator_reference, ip_type, analyst, source_method, add_domain=False, add_relationship=True, bucket_list=bucket_list, ticket=ticket, cache=cache) # run ip triage if is_item_new and is_validate_only == False: ip_object.reload() run_triage(None, ip_object, analyst) retVal['success'] = True retVal['object'] = ip_object return retVal
def migrate_1_to_2(self): """ Migrate from schema 1 to 2. """ if self.schema_version < 1: migrate_0_to_1(self) if self.schema_version == 1: old_analysis = getattr(self.unsupported_attrs, 'old_analysis', None) self.activity = [] self.campaign = [] if old_analysis: # activity if 'activity' in old_analysis: for a in old_analysis['activity']: (analyst, description) = ('', '') (date, start_date, end_date) = (None, None, None) if 'analyst' in a: analyst = a['analyst'] if 'description' in a: description = a['description'] if 'date' in a: date = a['date'] if 'start_date' in a: start_date = a['start_date'] if 'end_date' in a: end_date = a['end_date'] self.add_activity(analyst=analyst, start_date=start_date, end_date=end_date, date=date, description=description) # campaign if 'campaign' in old_analysis: for c in old_analysis['campaign']: (analyst, description) = ('', '') (date, confidence, name) = (None, 'low', '') if not 'analyst' in c: c['analyst'] = analyst if not 'description' in c: c['description'] = description if not 'date' in c: c['date'] = date if not 'confidence' in c: c['confidence'] = confidence if not 'name' in c: c['name'] = name ec = EmbeddedCampaign(analyst=c['analyst'], description=c['description'], date=c['date'], confidence=c['confidence'], name=c['name']) self.add_campaign(ec) # confidence if 'confidence' in old_analysis: confidence = old_analysis['confidence'] (analyst, rating) = ('', 'unknown') if 'analyst' in confidence: analyst = confidence['analyst'] if 'rating' in confidence: rating = confidence['rating'] self.set_confidence(analyst=analyst, rating=rating) # impact if 'impact' in old_analysis: impact = old_analysis['impact'] (analyst, rating) = ('', 'unknown') if 'analyst' in impact: analyst = impact['analyst'] if 'rating' in impact: rating = impact['rating'] self.set_impact(analyst=analyst, rating=rating) self.schema_version = 2
def upsert_domain(domain, source, username=None, campaign=None, confidence=None, bucket_list=None, ticket=None, cache={}, related_id=None, related_type=None, relationship_type=None): """ Add or update a domain/FQDN. Campaign is assumed to be a list of campaign dictionary objects. :param domain: The domain to add/update. :type domain: str :param source: The name of the source. :type source: str :param username: The user adding/updating the domain. :type username: str :param campaign: The campaign to attribute to this domain. :type campaign: list, str :param confidence: Confidence for the campaign attribution. :type confidence: str :param bucket_list: List of buckets to add to this domain. :type bucket_list: list, str :param ticket: The ticket for this domain. :type ticket: str :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_id: str :param relationship_type: Type of relationship to create. :type relationship_type: str :returns: dict with keys: "success" (boolean), "object" the domain that was added, "is_domain_new" (boolean) """ # validate domain and grab root domain (root, domain, error) = get_valid_root_domain(domain) if error: return {'success': False, 'message': error} is_fqdn_domain_new = False is_root_domain_new = False if not campaign: campaign = [] # assume it's a list, but check if it's a string elif isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username) campaign = [c] # assume it's a list, but check if it's a string if isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = '' instance.method = '' instance.analyst = username instance.date = datetime.datetime.now() s.instances = [instance] source = [s] fqdn_domain = None root_domain = None cached_results = cache.get(form_consts.Domain.CACHED_RESULTS) if cached_results != None: if domain != root: fqdn_domain = cached_results.get(domain) root_domain = cached_results.get(root) else: root_domain = cached_results.get(root) else: #first find the domain(s) if it/they already exist root_domain = Domain.objects(domain=root).first() if domain != root: fqdn_domain = Domain.objects(domain=domain).first() #if they don't exist, create them if not root_domain: root_domain = Domain() root_domain.domain = root root_domain.source = [] root_domain.record_type = 'A' is_root_domain_new = True if cached_results != None: cached_results[root] = root_domain if domain != root and not fqdn_domain: fqdn_domain = Domain() fqdn_domain.domain = domain fqdn_domain.source = [] fqdn_domain.record_type = 'A' is_fqdn_domain_new = True if cached_results != None: cached_results[domain] = fqdn_domain # if new or found, append the new source(s) for s in source: if root_domain: root_domain.add_source(s) if fqdn_domain: fqdn_domain.add_source(s) #campaigns #both root and fqdn get campaigns updated for c in campaign: if root_domain: root_domain.add_campaign(c) if fqdn_domain: fqdn_domain.add_campaign(c) if username: if root_domain: root_domain.analyst = username if fqdn_domain: fqdn_domain.analyst = username if bucket_list: if root_domain: root_domain.add_bucket_list(bucket_list, username) if fqdn_domain: fqdn_domain.add_bucket_list(bucket_list, username) if ticket: if root_domain: root_domain.add_ticket(ticket, username) if fqdn_domain: fqdn_domain.add_ticket(ticket, username) related_obj = None if related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal # save try: if root_domain: root_domain.save(username=username) if fqdn_domain: fqdn_domain.save(username=username) except Exception, e: return {'success': False, 'message': e}
def ip_add_update(ip_address, ip_type, source=None, source_method='', source_reference='', source_tlp=None, campaign=None, confidence='low', user=None, is_add_indicator=False, indicator_reference='', bucket_list=None, ticket=None, is_validate_only=False, cache={}, related_id=None, related_type=None, relationship_type=None, description=''): """ Add/update an IP address. :param ip_address: The IP to add/update. :type ip_address: str :param ip_type: The type of IP this is. :type ip_type: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param campaign: A campaign to attribute to this IP address. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param user: The user adding/updating this IP. :type user: str :param is_add_indicator: Also add an Indicator for this IP. :type is_add_indicator: bool :param indicator_reference: Reference for the indicator. :type indicator_reference: str :param bucket_list: Buckets to assign to this IP. :type bucket_list: str :param ticket: Ticket to assign to this IP. :type ticket: str :param is_validate_only: Only validate, do not add/update. :type is_validate_only: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_type: str :param relationship_type: Type of relationship to create. :type relationship_type: str :param description: A description for this IP :type description: str :returns: dict with keys: "success" (boolean), "message" (str), "object" (if successful) :class:`crits.ips.ip.IP` """ if not source: return {"success": False, "message": "Missing source information."} source_name = source (ip_address, error) = validate_and_normalize_ip(ip_address, ip_type) if error: return {"success": False, "message": error} retVal = {} is_item_new = False ip_object = None cached_results = cache.get(form_consts.IP.CACHED_RESULTS) if cached_results != None: ip_object = cached_results.get(ip_address) else: ip_object = IP.objects(ip=ip_address).first() if not ip_object: ip_object = IP() ip_object.ip = ip_address ip_object.ip_type = ip_type is_item_new = True if cached_results != None: cached_results[ip_address] = ip_object if not ip_object.description: ip_object.description = description or '' elif ip_object.description != description: ip_object.description += "\n" + (description or '') if isinstance(source_name, basestring): if user.check_source_write(source): source = [ create_embedded_source(source, reference=source_reference, method=source_method, tlp=source_tlp, analyst=user.username) ] else: return { "success": False, "message": "User does not have permission to add object \ using source %s." % source } if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user.username) campaign = [c] if campaign: for camp in campaign: ip_object.add_campaign(camp) if source: for s in source: ip_object.add_source(s) else: return {"success": False, "message": "Missing source information."} if bucket_list: ip_object.add_bucket_list(bucket_list, user.username) if ticket: ip_object.add_ticket(ticket, user.username) related_obj = None if related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip]) if is_validate_only == False: ip_object.save(username=user.username) #set the URL for viewing the new data if is_item_new == True: retVal['message'] = ('Success! Click here to view the new IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) else: message = ('Updated existing IP: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message elif is_validate_only == True: if ip_object.id != None and is_item_new == False: message = ('Warning: IP already exists: ' '<a href="%s">%s</a>' % (resp_url, ip_object.ip)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message if is_add_indicator: from crits.indicators.handlers import handle_indicator_ind result = handle_indicator_ind(ip_address, source_name, ip_type, IndicatorThreatTypes.UNKNOWN, IndicatorAttackTypes.UNKNOWN, user, source_method=source_method, source_reference=indicator_reference, source_tlp=source_tlp, add_domain=False, add_relationship=True, bucket_list=bucket_list, ticket=ticket, cache=cache) if related_obj and ip_object and relationship_type: relationship_type = RelationshipTypes.inverse( relationship=relationship_type) ip_object.add_relationship(related_obj, relationship_type, analyst=user.username, get_rels=False) ip_object.save(username=user.username) # run ip triage if is_item_new and is_validate_only == False: ip_object.reload() run_triage(ip_object, user) retVal['success'] = True retVal['object'] = ip_object return retVal
def add_new_event(title, description, event_type, source, method, reference, date, analyst, bucket_list=None, ticket=None, campaign=None, campaign_confidence=None, related_id=None, related_type=None, relationship_type=None): """ Add a new Event to CRITs. :param title: Event title. :type title: str :param description: Event description. :type description: str :param event_type: Event type. :type event_type: str :param source: The source which provided this information. :type source: str :param method: THe method of acquiring this information. :type method: str :param reference: Reference to this data. :type reference: str :param date: Date of acquiring this data. :type date: datetime.datetime :param analyst: The user adding this Event. :type analyst: str :param bucket_list: The bucket(s) to associate with this Event. :type: str :param ticket: Ticket to associate with this event. :type ticket: str :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_type: str :param relationship_type: Type of relationship to create. :type relationship_type: str :returns: dict with keys "success" (boolean) and "message" (str) :param campaign: Campaign to associate with this Event :type campaign: str :param campaign_confidence: Confidence to associate with the Campaign :type campaign_confidence: str """ result = dict() if not source: return {'success': False, 'message': "Missing source information."} event = Event() event.title = title event.description = description event.set_event_type(event_type) s = create_embedded_source(name=source, reference=reference, method=method, analyst=analyst, date=date) event.add_source(s) valid_campaign_confidence = { 'low': 'low', 'medium': 'medium', 'high': 'high' } valid_campaigns = {} for c in Campaign.objects(active='on'): valid_campaigns[c['name'].lower()] = c['name'] if campaign: if isinstance(campaign, basestring) and len(campaign) > 0: if campaign.lower() not in valid_campaigns: result = { 'success': False, 'message': '{} is not a valid campaign.'.format(campaign) } else: confidence = valid_campaign_confidence.get( campaign_confidence, 'low') campaign = EmbeddedCampaign(name=campaign, confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now()) event.add_campaign(campaign) if bucket_list: event.add_bucket_list(bucket_list, analyst) if ticket: event.add_ticket(ticket, analyst) related_obj = None if related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal try: event.save(username=analyst) if related_obj and event and relationship_type: relationship_type = RelationshipTypes.inverse( relationship=relationship_type) event.add_relationship(related_obj, relationship_type, analyst=analyst, get_rels=False) event.save(username=analyst) # run event triage event.reload() run_triage(event, analyst) message = ( '<div>Success! Click here to view the new event: <a href=' '"%s">%s</a></div>' % (reverse('crits.events.views.view_event', args=[event.id]), title)) result = { 'success': True, 'message': message, 'id': str(event.id), 'object': event } except ValidationError, e: result = {'success': False, 'message': e}
def add_object(type_, oid, object_type, name, source, method, reference, analyst, value=None, file_=None, add_indicator=False, get_objects=True, indicator_campaign=None, indicator_campaign_confidence=None, obj=None, is_sort_relationships=False, is_validate_only=False, is_validate_locally=False, cache={}): """ Add an object to the database. :param type_: The top-level object type. :type type_: str :param oid: The ObjectId of the top-level object. :type oid: str :param object_type: The type of the ObjectType being added. :type object_type: str :param name: The name of the ObjectType being added. :type name: str :param source: The name of the source adding this object. :type source: str :param method: The method for this object. :type method: str :param reference: The reference for this object. :type reference: str :param analyst: The user adding this object. :type analyst: str :param value: The value of the object. :type value: str :param file_: The file if the object is a file upload. :type file_: file handle. :param add_indicator: Also add an indicator for this object. :type add_indicator: bool :param get_objects: Return the formatted list of objects when completed. :type get_object: bool :param is_validate_only: Only validate, do not add. :type is_validate_only: bool :param is_validate_locally: Only validate, do not add. :type is_validate_locally: bool :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :param obj: The CRITs top-level object we are adding objects to. This is an optional parameter used mainly for performance reasons (by not querying mongo if we already have the top level-object). :type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes` :returns: dict with keys: "success" (boolean), "message" (str), "objects" (list), "relationships" (list) """ results = {} if oid == None: oid = "" if obj == None: obj = class_from_id(type_, oid) if not obj: if is_validate_locally == True: # TODO: Perform some form of validation results['success'] = True return results else: results['message'] = "Could not find item to add object to." results['success'] = False return results try: cur_len = len(obj.obj) if file_: data = file_.read() filename = file_.name md5sum = md5(data).hexdigest() value = md5sum reference = filename obj.add_object(object_type, name, value, source, method, reference, analyst) if is_validate_only == False: obj.save(username=analyst) new_len = len(obj.obj) if new_len > cur_len: results['message'] = "Object added successfully!" results['success'] = True if file_: # do we have a pcap? if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1', '\x0a\x0d\x0d\x0a'): handle_pcap_file(filename, data, source, user=analyst, related_id=oid, related_type=type_) else: #XXX: MongoEngine provides no direct GridFS access so we # need to use pymongo directly. col = settings.COL_OBJECTS grid = mongo_connector("%s.files" % col) if grid.find({'md5': md5sum}).count() == 0: put_file(filename, data, collection=col) if add_indicator and is_validate_only == False: from crits.indicators.handlers import handle_indicator_ind if object_type != name: object_type = "%s - %s" % (object_type, name) ind_res = handle_indicator_ind( value, source, reference, object_type, analyst, method=method, add_domain=True, campaign=indicator_campaign, campaign_confidence=indicator_campaign_confidence, cache=cache) if ind_res['success']: ind = ind_res['object'] # Inherit campaigns from top level item when creating # an indicator from an object if no campaigns were specified if indicator_campaign == None and ind != None: for campaign in obj.campaign: ec = EmbeddedCampaign( name=campaign.name, confidence=campaign.confidence, description="", analyst=analyst, date=datetime.datetime.now()) ind.add_campaign(ec) ind.save(username=analyst) forge_relationship(left_class=obj, right_class=ind, rel_type="Related_To", analyst=analyst, get_rels=is_sort_relationships) if is_sort_relationships == True: if file_ or add_indicator: # does this line need to be here? # obj.reload() results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results['relationships'] = obj.sort_relationships( analyst, meta=True) else: results[ 'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] " results['success'] = False if (get_objects): results['objects'] = obj.sort_objects() return results except ValidationError, e: return {'success': False, 'message': e}
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}): """ Add a new domain to CRITs. :param data: The data about the domain. :type data: dict :param request: The Django request. :type request: :class:`django.http.HttpRequest` :param errors: A list of current errors to append to. :type errors: list :param rowData: Any objects that need to be added to the domain. :type rowData: dict :param is_validate_only: Only validate the data and return any errors. :type is_validate_only: boolean :param cache: Cached data, typically for performance enhancements during bulk operations. :type cache: dict :returns: tuple (<result>, <errors>, <retVal>) """ result = False retVal = {} domain = data['domain'] add_ip = data.get('add_ip') ip = data.get('ip') ip_type = data.get('ip_type') if add_ip: error = validate_and_normalize_ip(ip, ip_type)[1] if error: errors.append(error) if is_validate_only: error = get_valid_root_domain(domain)[2] if error: errors.append(error) # check for duplicate domains fqdn_domain = retrieve_domain(domain, cache) if fqdn_domain: if isinstance(fqdn_domain, Domain): resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) message = ('Warning: Domain already exists: ' '<a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message else: result_cache = cache.get(form_consts.Domain.CACHED_RESULTS) result_cache[domain.lower()] = True elif not errors: username = request.user.username reference = data.get('domain_reference') source_name = data.get('domain_source') method = data.get('domain_method') source = [ create_embedded_source(source_name, reference=reference, method=method, analyst=username) ] bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME) if data.get('campaign') and data.get('confidence'): campaign = [ EmbeddedCampaign(name=data.get('campaign'), confidence=data.get('confidence'), analyst=username) ] else: campaign = [] retVal = upsert_domain(domain, source, username, campaign, bucket_list=bucket_list, ticket=ticket, cache=cache) if not retVal['success']: errors.append(retVal.get('message')) retVal['message'] = "" else: new_domain = retVal['object'] ip_result = {} if add_ip: if data.get('same_source'): ip_source = source_name ip_method = method ip_reference = reference else: ip_source = data.get('ip_source') ip_method = data.get('ip_method') ip_reference = data.get('ip_reference') from crits.ips.handlers import ip_add_update ip_result = ip_add_update(ip, ip_type, ip_source, ip_method, ip_reference, campaign=campaign, analyst=username, bucket_list=bucket_list, ticket=ticket, cache=cache) if not ip_result['success']: errors.append(ip_result['message']) else: #add a relationship with the new IP address new_ip = ip_result['object'] if new_domain and new_ip: new_domain.add_relationship(rel_item=new_ip, rel_type='Resolved_To', analyst=username, get_rels=False) new_domain.save(username=username) new_ip.save(username=username) #set the URL for viewing the new data resp_url = reverse('crits.domains.views.domain_detail', args=[domain]) if retVal['is_domain_new'] == True: retVal['message'] = ( 'Success! Click here to view the new domain: ' '<a href="%s">%s</a>' % (resp_url, domain)) else: message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain)) retVal['message'] = message retVal[form_consts.Status. STATUS_FIELD] = form_consts.Status.DUPLICATE retVal['warning'] = message #add indicators if data.get('add_indicators'): from crits.indicators.handlers import create_indicator_from_tlo # If we have an IP object, add an indicator for that. if ip_result.get('success'): ip = ip_result['object'] result = create_indicator_from_tlo('IP', ip, username, ip_source, add_domain=False) ip_ind = result.get('indicator') if not result['success']: errors.append(result['message']) # Add an indicator for the domain. result = create_indicator_from_tlo('Domain', new_domain, username, source_name, add_domain=False) if not result['success']: errors.append(result['message']) elif ip_result.get('success') and ip_ind: forge_relationship(left_class=result['indicator'], right_class=ip_ind, rel_type='Resolved_To', analyst=username) result = True # This block validates, and may also add, objects to the Domain if retVal.get('success') or is_validate_only == True: if rowData: objectsData = rowData.get(form_consts.Common.OBJECTS_DATA) # add new objects if they exist if objectsData: objectsData = json.loads(objectsData) current_domain = retrieve_domain(domain, cache) for object_row_counter, objectData in enumerate( objectsData, 1): if current_domain != None: # if the domain exists then try to add objects to it if isinstance(current_domain, Domain) == True: objectDict = object_array_to_dict( objectData, "Domain", current_domain.id) else: objectDict = object_array_to_dict( objectData, "Domain", "") current_domain = None else: objectDict = object_array_to_dict( objectData, "Domain", "") (obj_result, errors, obj_retVal) = validate_and_add_new_handler_object( None, objectDict, request, errors, object_row_counter, is_validate_only=is_validate_only, cache=cache, obj=current_domain) if not obj_result: retVal['success'] = False return result, errors, retVal
def handle_indicator_insert(ind, source, reference='', analyst='', method='', add_domain=False, add_relationship=False, cache={}): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ is_new_indicator = False rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4} indicator = Indicator.objects(ind_type=ind['type'], value=ind['value']).first() if not indicator: indicator = Indicator() indicator.ind_type = ind['type'] indicator.value = ind['value'] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) is_new_indicator = True ec = None if 'campaign' in ind: confidence = 'low' if 'campaign_confidence' in ind: confidence = ind['campaign_confidence'] ec = EmbeddedCampaign(name=ind['campaign'], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now()) if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get( indicator.confidence.rating, 0): indicator.confidence.rating = ind['confidence'] indicator.confidence.analyst = analyst if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get( indicator.impact.rating, 0): indicator.impact.rating = ind['impact'] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if ec: indicator.add_campaign(ec) indicator.save(username=analyst) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.value if ind_type in ("URI - Domain Name", "URI - URL"): if ind_type == "URI - URL": domain = ind_value.split("/")[2] elif ind_type == "URI - Domain Name": domain = ind_value #try: (sdomain, fqdn) = get_domain(domain) success = None if add_domain: success = upsert_domain(sdomain, fqdn, indicator.source, '%s' % analyst, None, bucket_list=bucket_list, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: dmain = Domain.objects(domain=domain).first() else: dmain = success['object'] if dmain: dmain.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) indicator.save(username=analyst) elif ind_type.startswith( "Address - ip") or ind_type == "Address - cidr": success = None if add_domain: success = ip_add_update(indicator.value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache) if not success['success']: return {'success': False, 'message': success['message']} if not success or not 'object' in success: ip = IP.objects(ip=indicator.value).first() else: ip = success['object'] if ip: ip.add_relationship(rel_item=indicator, rel_type='Related_To', analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) indicator.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(None, indicator, analyst) return { 'success': True, 'objectid': indicator.id, 'is_new_indicator': is_new_indicator, 'object': indicator }
def add_sample_for_event(event_id, data, analyst, filedata=None, filename=None, md5=None, email_addr=None, inherit_sources=False): """ Add a sample related to this Event. :param event_id: The ObjectId of the Event to associate with. :type event_id: str :param data: The form data. :type data: dict :param analyst: The user adding this Sample. :type analyst: str :param filedata: The sample data. :type filedata: file handle. :param filename: The name of the file. :type filename: str :param md5: The MD5 of the file. :type md5: str :param email_addr: Email address to which to email the sample :type email_addr: str :param inherit_sources: 'True' if Sample should inherit Event's Source(s) :type inherit_sources: bool :returns: dict with keys "success" (boolean) and "message" (str) """ response = { 'success': False, 'message': 'Unknown error; unable to upload file.' } users_sources = user_sources(analyst) event = Event.objects(id=event_id, source__name__in=users_sources).first() if not event: return {'success': False, 'message': "No matching event found"} source = data['source'] reference = data['reference'] file_format = data['file_format'] campaign = data['campaign'] confidence = data['confidence'] bucket_list = data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] ticket = data[form_consts.Common.TICKET_VARIABLE_NAME] method = data['method'] if filename: filename = filename.strip() # If selected, new sample inherits the campaigns of the related event. if data['inherit_campaigns']: if campaign: event.campaign.append( EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)) campaign = event.campaign inherited_source = event.source if inherit_sources else None try: if filedata: result = handle_uploaded_file(filedata, source, method, reference, file_format, data['password'], analyst, campaign, confidence, related_id=event.id, related_type='Event', filename=filename, bucket_list=bucket_list, ticket=ticket, inherited_source=inherited_source) else: if md5: md5 = md5.strip().lower() result = handle_uploaded_file(None, source, method, reference, file_format, None, analyst, campaign, confidence, related_id=event.id, related_type='Event', filename=filename, md5=md5, bucket_list=bucket_list, ticket=ticket, inherited_source=inherited_source, is_return_only_md5=False) except ZipFileError, zfe: return {'success': False, 'message': zfe.value}
def add_new_exploit(name, cve=None, description=None, source=None, source_method=None, source_reference=None, source_tlp=None, campaign=None, confidence=None, user=None, bucket_list=None, ticket=None, related_id=None, related_type=None, relationship_type=None): """ Add an Exploit to CRITs. :param name: The name of the exploit. :type name: str :param cve: CVE for the exploit. :type cve: str :param description: Description of the exploit. :type description: str :param source: Name of the source which provided this information. :type source: str :param source_method: Method of acquiring this data. :type source_method: str :param source_reference: A reference to this data. :type source_reference: str :param source_tlp: TLP value for this source :type source_reference: str :param campaign: A campaign to attribute to this exploit. :type campaign: str :param confidence: Confidence level in the campaign attribution. :type confidence: str ("low", "medium", "high") :param user: The user adding this exploit. :type user: str :param bucket_list: Buckets to assign to this exploit. :type bucket_list: str :param ticket: Ticket to assign to this exploit. :type ticket: str :param related_id: ID of object to create relationship with :type related_id: str :param related_type: Type of object to create relationship with :type related_type: str :param relationship_type: Type of relationship to create. :type relationship_type: str :returns: dict with keys: "success" (boolean), "message" (str), "object" (if successful) :class:`crits.exploits.exploit.Exploit` """ is_item_new = False retVal = {} exploit = Exploit.objects(name=name).first() if not exploit: exploit = Exploit() exploit.name = name if description: exploit.description = description.strip() is_item_new = True username = user.username if isinstance(source, basestring): if user.check_source_write(source): source = [ create_embedded_source(source, reference=source_reference, method=source_method, tlp=source_tlp, analyst=username) ] else: return { "success": False, "message": "User does not have permission to add object \ using source %s." % source } elif isinstance(source, EmbeddedSource): source = [source] if isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username) campaign = [c] if campaign: for camp in campaign: exploit.add_campaign(camp) if source: for s in source: exploit.add_source(s) else: return {"success": False, "message": "Missing source information."} exploit.cve = cve.strip() if bucket_list: exploit.add_bucket_list(bucket_list, user) if ticket: exploit.add_ticket(ticket, user) related_obj = None if related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal exploit.save(username=username) if related_obj and exploit and relationship_type: relationship_type = RelationshipTypes.inverse( relationship=relationship_type) exploit.add_relationship(related_obj, relationship_type, analyst=user, get_rels=False) exploit.save(username=username) # run exploit triage if is_item_new: exploit.reload() run_triage(exploit, username) resp_url = reverse('crits.exploits.views.exploit_detail', args=[exploit.id]) retVal['message'] = ('Success! Click here to view the new Exploit: ' '<a href="%s">%s</a>' % (resp_url, exploit.name)) retVal['success'] = True retVal['object'] = exploit retVal['id'] = str(exploit.id) return retVal
def upsert_domain(sdomain, domain, source, username=None, campaign=None, confidence=None, bucket_list=None, ticket=None, cache={}): """ Add or update a domain/FQDN. Campaign is assumed to be a list of campaign dictionary objects. :param sdomain: Response from parsing the domain for a root domain. Will either be an error message or the root domain itself. :type sdomain: str :param domain: The domain to add/update. :type domain: str :param source: The name of the source. :type source: str :param username: The user adding/updating the domain. :type username: str :param campaign: The campaign to attribute to this domain. :type campaign: list, str :param confidence: Confidence for the campaign attribution. :type confidence: str :param bucket_list: List of buckets to add to this domain. :type bucket_list: list, str :param ticket: The ticket for this domain. :type ticket: str :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "object" the domain that was added, "is_domain_new" (boolean) """ if sdomain == "no_tld_found_error": #oops... return {'success': False, 'message': "Invalid domain: %s " % sdomain} is_fqdn_domain_new = False is_root_domain_new = False if not campaign: campaign = [] # assume it's a list, but check if it's a string elif isinstance(campaign, basestring): c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username) campaign = [c] # assume it's a list, but check if it's a string if isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = '' instance.method = '' instance.analyst = username instance.date = datetime.datetime.now() s.instances = [instance] source = [s] fqdn_domain = None root_domain = None cached_results = cache.get(form_consts.Domain.CACHED_RESULTS) if cached_results != None: if domain != sdomain: fqdn_domain = cached_results.get(domain) root_domain = cached_results.get(sdomain) else: root_domain = cached_results.get(sdomain) else: #first find the domain(s) if it/they already exist root_domain = Domain.objects(domain=sdomain).first() if domain != sdomain: fqdn_domain = Domain.objects(domain=domain).first() #if they don't exist, create them if not root_domain: root_domain = Domain() root_domain.domain = sdomain.strip() root_domain.source = [] root_domain.record_type = 'A' is_root_domain_new = True if cached_results != None: cached_results[sdomain] = root_domain if domain != sdomain and not fqdn_domain: fqdn_domain = Domain() fqdn_domain.domain = domain.strip() fqdn_domain.source = [] fqdn_domain.record_type = 'A' is_fqdn_domain_new = True if cached_results != None: cached_results[domain] = fqdn_domain # if new or found, append the new source(s) for s in source: if root_domain: root_domain.add_source(s) if fqdn_domain: fqdn_domain.add_source(s) #campaigns #both root and fqdn get campaigns updated for c in campaign: if root_domain: root_domain.add_campaign(c) if fqdn_domain: fqdn_domain.add_campaign(c) if username: if root_domain: root_domain.analyst = username if fqdn_domain: fqdn_domain.analyst = username if bucket_list: if root_domain: root_domain.add_bucket_list(bucket_list, username) if fqdn_domain: fqdn_domain.add_bucket_list(bucket_list, username) if ticket: if root_domain: root_domain.add_ticket(ticket, username) if fqdn_domain: fqdn_domain.add_ticket(ticket, username) # save try: if root_domain: root_domain.save(username=username) if fqdn_domain: fqdn_domain.save(username=username) except Exception, e: return {'success': False, 'message': e}
def upload_child(request, parent_md5): """ Upload a new child sample. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :param parent_md5: The MD5 of the parent sample. :type parent_md5: str :returns: :class:`django.http.HttpResponse` """ new_samples = [] if request.method == "POST": form = EmailAttachForm(request.user.username, request.POST, request.FILES) if form.is_valid(): if request.FILES or 'filename' in request.POST and 'md5' in request.POST: # Child samples inherit all of the sources of the parent. parent = Sample.objects(md5=parent_md5).first() if not parent: return render_to_response( 'error.html', {'error': "Unable to find parent."}, RequestContext(request)) source = parent.source campaign_name = request.POST['campaign'] confidence = request.POST['confidence'] parent.campaign.append( EmbeddedCampaign(name=campaign_name, confidence=confidence, analyst=request.user.username)) campaigns = parent.campaign try: if request.FILES: new_samples = handle_uploaded_file( request.FILES["filedata"], source, None, form.cleaned_data["file_format"], form.cleaned_data["password"], user=request.user.username, campaign=campaigns, parent_md5=parent_md5, bucket_list=form.cleaned_data[ form_consts.Common.BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME]) else: filename = request.POST['filename'].strip() md5 = request.POST['md5'].strip().lower() if not filename or not md5: error = "Need a file, or a filename and an md5." return render_to_response('error.html', {'error': error}, RequestContext(request)) else: new_samples = handle_uploaded_file( None, source, None, form.cleaned_data["file_format"], form.cleaned_data["password"], user=request.user.username, campaign=campaigns, parent_md5=parent_md5, filename=filename, bucket_list=form.cleaned_data[ form_consts.Common. BUCKET_LIST_VARIABLE_NAME], ticket=form.cleaned_data[ form_consts.Common.TICKET_VARIABLE_NAME], md5=md5) except ZipFileError, zfe: return render_to_response('error.html', {'error': zfe.value}, RequestContext(request)) else: return render_to_response( 'error.html', {'error': "Need a file, or a filename and an md5."}, RequestContext(request)) else: return render_to_response('error.html', {'error': 'form error'}, RequestContext(request))