def process_bulk_add_email_addresses(request, formdict): """ Performs the bulk add of email addreesses by parsing the request data. Batches some data into a cache object for performance by reducing large amounts of single database queries. :param request: Django request. :type request: :class:`django.http.HttpRequest` :param formdict: The form representing the bulk uploaded data. :type formdict: dict :returns: :class:`django.http.HttpResponse` """ email_names = [] cached_results = {} cleanedRowsData = convert_handsontable_to_rows(request) for rowData in cleanedRowsData: if rowData != None and rowData.get(form_consts.EmailAddress.EMAIL_ADDRESS) != None: email_names.append(rowData.get(form_consts.EmailAddress.EMAIL_ADDRESS).lower()) email_results = EmailAddress.objects(address__in=email_names) for email_result in email_results: cached_results[email_result.address] = email_result cache = {form_consts.EmailAddress.CACHED_RESULTS: cached_results, 'cleaned_rows_data': cleanedRowsData} response = parse_bulk_upload(request, parse_row_to_bound_email_form, add_new_email_via_bulk, formdict, cache) return response
def process_bulk_add_email_addresses(request, formdict): """ Performs the bulk add of email addreesses by parsing the request data. Batches some data into a cache object for performance by reducing large amounts of single database queries. :param request: Django request. :type request: :class:`django.http.HttpRequest` :param formdict: The form representing the bulk uploaded data. :type formdict: dict :returns: :class:`django.http.HttpResponse` """ email_names = [] cached_results = {} cleanedRowsData = convert_handsontable_to_rows(request) for rowData in cleanedRowsData: if rowData != None and rowData.get( form_consts.EmailAddress.EMAIL_ADDRESS) != None: email_names.append( rowData.get(form_consts.EmailAddress.EMAIL_ADDRESS).lower()) email_results = EmailAddress.objects(address__in=email_names) for email_result in email_results: cached_results[email_result.address] = email_result cache = { form_consts.EmailAddress.CACHED_RESULTS: cached_results, 'cleaned_rows_data': cleanedRowsData } response = parse_bulk_upload(request, parse_row_to_bound_email_form, add_new_email_via_bulk, formdict, cache) return response
def class_from_id(type_, _id): """ Return an instantiated class object. :param type_: The CRIPTs top-level object type. :type type_: str :param _id: The ObjectId to search for. :type _id: str :returns: class which inherits from :class:`cripts.core.cripts_mongoengine.CriptsBaseAttributes` """ #Quick fail if not _id or not type_: return None # doing this to avoid circular imports from cripts.comments.comment import Comment from cripts.core.cripts_mongoengine import Action from cripts.core.source_access import SourceAccess from cripts.core.user_role import UserRole from cripts.events.event import Event from cripts.usernames.username import UserName from cripts.targets.target import Target from cripts.hashes.hash import Hash from cripts.datasets.dataset import Dataset from cripts.email_addresses.email_address import EmailAddress # make sure it's a string _id = str(_id) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if not ObjectId.is_valid(_id.decode('utf8')): return None if type_ == 'Comment': return Comment.objects(id=_id).first() elif type_ == 'Event': return Event.objects(id=_id).first() elif type_ == 'Action': return Action.objects(id=_id).first() elif type_ == 'SourceAccess': return SourceAccess.objects(id=_id).first() elif type_ == 'UserRole': return UserRole.objects(id=_id).first() elif type_ == 'UserName': return UserName.objects(id=_id).first() elif type_ == 'Target': return Target.objects(id=_id).first() elif type_ == 'Hash': return Hash.objects(id=_id).first() elif type_ == 'Dataset': return Dataset.objects(id=_id).first() elif type_ == 'EmailAddress': return EmailAddress.objects(id=_id).first() else: return None
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRIPTs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`cripts.core.cripts_mongoengine.CriptsBaseAttributes` """ #Quick fail if not type_ or not value: return None # doing this to avoid circular imports from cripts.comments.comment import Comment from cripts.events.event import Event from cripts.usernames.username import UserName from cripts.targets.target import Target from cripts.hashes.hash import Hash from cripts.datasets.dataset import Dataset from cripts.email_addresses.email_address import EmailAddress # Make sure value is a string... value = str(value) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if (type_ in [ 'Comment', 'Event', 'UserName', 'Target', 'Hash', 'Dataset', 'EmailAddress' ] and not ObjectId.is_valid(value.decode('utf8'))): return None if type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'UserName': return UserName.objects(id=value).first() elif type_ == 'Target': return Target.objects(id=value).first() elif type_ == 'Hash': return Hash.objects(id=value).first() elif type_ == 'Dataset': return Dataset.objects(id=value).first() elif type_ == 'EmailAddress': return EmailAddress.objects(id=value).first() else: return None
def class_from_value(type_, value): """ Return an instantiated class object. :param type_: The CRIPTs top-level object type. :type type_: str :param value: The value to search for. :type value: str :returns: class which inherits from :class:`cripts.core.cripts_mongoengine.CriptsBaseAttributes` """ #Quick fail if not type_ or not value: return None # doing this to avoid circular imports from cripts.comments.comment import Comment from cripts.events.event import Event from cripts.usernames.username import UserName from cripts.targets.target import Target from cripts.hashes.hash import Hash from cripts.datasets.dataset import Dataset from cripts.email_addresses.email_address import EmailAddress # Make sure value is a string... value = str(value) # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise # the queries below will raise a ValidationError exception. if (type_ in ['Comment','Event','UserName','Target','Hash','Dataset','EmailAddress'] and not ObjectId.is_valid(value.decode('utf8'))): return None if type_ == 'Comment': return Comment.objects(id=value).first() elif type_ == 'Event': return Event.objects(id=value).first() elif type_ == 'UserName': return UserName.objects(id=value).first() elif type_ == 'Target': return Target.objects(id=value).first() elif type_ == 'Hash': return Hash.objects(id=value).first() elif type_ == 'Dataset': return Dataset.objects(id=value).first() elif type_ == 'EmailAddress': return EmailAddress.objects(id=value).first() else: return None
def get_email_address_details(address, analyst): """ Generate the data to render the Email Address details template. :param address: The name of the Address to get details for. :type address: str :param analyst: The user requesting this information. :type analyst: str :returns: template (str), arguments (dict) """ template = None allowed_sources = user_sources(analyst) address_object = EmailAddress.objects(address=address, source__name__in=allowed_sources).first() if not address_object: error = ("Either no data exists for this email address" " or you do not have permission to view it.") template = "error.html" args = {'error': error} return template, args address_object.sanitize_sources(username="******" % analyst, sources=allowed_sources) # remove pending notifications for user remove_user_from_notification("%s" % analyst, address_object.id, 'EmailAddress') # subscription subscription = { 'type': 'EmailAddress', 'id': address_object.id, 'subscribed': is_user_subscribed("%s" % analyst, 'EmailAddress', address_object.id), } #objects objects = address_object.sort_objects() #relationships relationships = address_object.sort_relationships("%s" % analyst, meta=True) # relationship relationship = { 'type': 'EmailAddress', 'value': address_object.id } #comments comments = {'comments': address_object.get_comments(), 'url_key':address_object.address} # favorites favorite = is_user_favorite("%s" % analyst, 'EmailAddress', address_object.id) # services service_list = get_supported_services('EmailAddress') # analysis results service_results = address_object.get_analysis_results() args = {'email_address': address_object, 'objects': objects, 'relationships': relationships, 'comments': comments, 'favorite': favorite, 'relationship': relationship, 'subscription': subscription, 'address': address_object.address, 'service_list': service_list, 'service_results': service_results} return template, args
def email_address_add_update(address, description=None, source=None, method='', reference='', analyst=None, datasets=None, bucket_list=None, ticket=None, is_validate_only=False, cache={}, related_id=None, related_type=None, relationship_type=None): retVal = {} if not source: return {"success" : False, "message" : "Missing source information."} # Parse out the e-mail address. Return an error if it looks invalid, (aka missing the @, has whitespace, etc) try: if ' ' in address: raise ValueError local_name, domain_part = address.strip().split('@', 1) if len(local_name) == 0 or len(domain_part) == 0: raise ValueError # lowercase the domain name and recreate the e-mail address address = '@'.join([local_name, domain_part.lower()]) except ValueError: return {'success': False, 'message': "Invalid Email Address Format"} is_item_new = False email_object = None cached_results = cache.get(form_consts.EmailAddress.CACHED_RESULTS) if cached_results != None: email_object = cached_results.get(address) else: email_object = EmailAddress.objects(address=address).first() if not email_object: email_object = EmailAddress() email_object.address = address email_object.description = description email_object.local_name = local_name email_object.domain = domain_part.lower() is_item_new = True if cached_results != None: cached_results[address] = email_object if not email_object.description: email_object.description = description or '' elif email_object.description != description: if description: email_object.description += "\n" + (description or '') if isinstance(source, basestring): source = [create_embedded_source(source, reference=reference, method=method, analyst=analyst)] if source: for s in source: email_object.add_source(s) else: return {"success" : False, "message" : "Missing source information."} if bucket_list: email_object.add_bucket_list(bucket_list, analyst) if ticket: email_object.add_ticket(ticket, analyst) related_obj = None if related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal resp_url = reverse('cripts.email_addresses.views.email_address_detail', args=[email_object.address]) if is_validate_only == False: email_object.save(username=analyst) #set the URL for viewing the new data if is_item_new == True: # Update the email stats counts = mongo_connector(settings.COL_COUNTS) count_stats = counts.find_one({'name': 'counts'}) if not count_stats or ('counts' not in count_stats): count_stats = {'counts':{}} if 'Email Addresses' not in count_stats['counts']: count_stats['counts']['Email Addresses'] = 0 else: count_stats['counts']['Email Addresses'] = count_stats['counts']['Email Addresses'] + 1 counts.update({'name': "counts"}, {'$set': {'counts': count_stats['counts']}}, upsert=True) retVal['message'] = ('Success! Click here to view the new Email: ' '<a href="%s">%s</a>' % (resp_url, email_object.address)) else: message = ('Updated existing Email: ' '<a href="%s">%s</a>' % (resp_url, email_object.address)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message elif is_validate_only == True: if email_object.id != None and is_item_new == False: message = ('Warning: Email already exists: ' '<a href="%s">%s</a>' % (resp_url, email_object.address)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message if related_obj and email_object and relationship_type: relationship_type=RelationshipTypes.inverse(relationship=relationship_type) email_object.add_relationship(related_obj, relationship_type, analyst=analyst, get_rels=False) email_object.save(username=analyst) # run email triage if is_item_new and is_validate_only == False: email_object.reload() run_triage(email_object, analyst) retVal['success'] = True retVal['object'] = email_object return retVal
def get_email_address_details(address, analyst): """ Generate the data to render the Email Address details template. :param address: The name of the Address to get details for. :type address: str :param analyst: The user requesting this information. :type analyst: str :returns: template (str), arguments (dict) """ template = None allowed_sources = user_sources(analyst) address_object = EmailAddress.objects( address=address, source__name__in=allowed_sources).first() if not address_object: error = ("Either no data exists for this email address" " or you do not have permission to view it.") template = "error.html" args = {'error': error} return template, args address_object.sanitize_sources(username="******" % analyst, sources=allowed_sources) # remove pending notifications for user remove_user_from_notification("%s" % analyst, address_object.id, 'EmailAddress') # subscription subscription = { 'type': 'EmailAddress', 'id': address_object.id, 'subscribed': is_user_subscribed("%s" % analyst, 'EmailAddress', address_object.id), } #objects objects = address_object.sort_objects() #relationships relationships = address_object.sort_relationships("%s" % analyst, meta=True) # relationship relationship = {'type': 'EmailAddress', 'value': address_object.id} #comments comments = { 'comments': address_object.get_comments(), 'url_key': address_object.address } # favorites favorite = is_user_favorite("%s" % analyst, 'EmailAddress', address_object.id) # services service_list = get_supported_services('EmailAddress') # analysis results service_results = address_object.get_analysis_results() args = { 'email_address': address_object, 'objects': objects, 'relationships': relationships, 'comments': comments, 'favorite': favorite, 'relationship': relationship, 'subscription': subscription, 'address': address_object.address, 'service_list': service_list, 'service_results': service_results } return template, args
def email_address_add_update(address, description=None, source=None, method='', reference='', analyst=None, datasets=None, bucket_list=None, ticket=None, is_validate_only=False, cache={}, related_id=None, related_type=None, relationship_type=None): retVal = {} if not source: return {"success": False, "message": "Missing source information."} # Parse out the e-mail address. Return an error if it looks invalid, (aka missing the @, has whitespace, etc) try: if ' ' in address: raise ValueError local_name, domain_part = address.strip().split('@', 1) if len(local_name) == 0 or len(domain_part) == 0: raise ValueError # lowercase the domain name and recreate the e-mail address address = '@'.join([local_name, domain_part.lower()]) except ValueError: return {'success': False, 'message': "Invalid Email Address Format"} is_item_new = False email_object = None cached_results = cache.get(form_consts.EmailAddress.CACHED_RESULTS) if cached_results != None: email_object = cached_results.get(address) else: email_object = EmailAddress.objects(address=address).first() if not email_object: email_object = EmailAddress() email_object.address = address email_object.description = description email_object.local_name = local_name email_object.domain = domain_part.lower() is_item_new = True if cached_results != None: cached_results[address] = email_object if not email_object.description: email_object.description = description or '' elif email_object.description != description: if description: email_object.description += "\n" + (description or '') if isinstance(source, basestring): source = [ create_embedded_source(source, reference=reference, method=method, analyst=analyst) ] if source: for s in source: email_object.add_source(s) else: return {"success": False, "message": "Missing source information."} if bucket_list: email_object.add_bucket_list(bucket_list, analyst) if ticket: email_object.add_ticket(ticket, analyst) related_obj = None if related_id: related_obj = class_from_id(related_type, related_id) if not related_obj: retVal['success'] = False retVal['message'] = 'Related Object not found.' return retVal resp_url = reverse('cripts.email_addresses.views.email_address_detail', args=[email_object.address]) if is_validate_only == False: email_object.save(username=analyst) #set the URL for viewing the new data if is_item_new == True: # Update the email stats counts = mongo_connector(settings.COL_COUNTS) count_stats = counts.find_one({'name': 'counts'}) if not count_stats or ('counts' not in count_stats): count_stats = {'counts': {}} if 'Email Addresses' not in count_stats['counts']: count_stats['counts']['Email Addresses'] = 0 else: count_stats['counts']['Email Addresses'] = count_stats[ 'counts']['Email Addresses'] + 1 counts.update({'name': "counts"}, {'$set': { 'counts': count_stats['counts'] }}, upsert=True) retVal['message'] = ('Success! Click here to view the new Email: ' '<a href="%s">%s</a>' % (resp_url, email_object.address)) else: message = ('Updated existing Email: ' '<a href="%s">%s</a>' % (resp_url, email_object.address)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message elif is_validate_only == True: if email_object.id != None and is_item_new == False: message = ('Warning: Email already exists: ' '<a href="%s">%s</a>' % (resp_url, email_object.address)) retVal['message'] = message retVal['status'] = form_consts.Status.DUPLICATE retVal['warning'] = message if related_obj and email_object and relationship_type: relationship_type = RelationshipTypes.inverse( relationship=relationship_type) email_object.add_relationship(related_obj, relationship_type, analyst=analyst, get_rels=False) email_object.save(username=analyst) # run email triage if is_item_new and is_validate_only == False: email_object.reload() run_triage(email_object, analyst) retVal['success'] = True retVal['object'] = email_object return retVal