def target_user_stats():
"""
Generate targets from email To/CC fields, then generate divisions from
targets list.
No cleanup or logic is being done on the To/CC fields. If they are not
valid email addresses (user@domain), they do not get added as a target.
"""
mapcode = """
function () {
try {
this.to.forEach(function(z) {
emit(z.toLowerCase(), {count: 1});
});
} catch(err) {}
}
"""
reducecode = """
function(k,v) {
var count = 0;
v.forEach(function(v) {
count += v["count"];
});
return {count: count};
}
"""
m = Code(mapcode)
r = Code(reducecode)
results = Email.objects(to__exists=True).map_reduce(m, r, 'inline')
for result in results:
try:
targs = Target.objects(email_address__iexact=result.key)
if not targs:
targs = [Target()]
targs[0].email_address = result.key.strip().lower()
for targ in targs:
targ.email_count = result.value['count']
targ.save()
except:
pass
mapcode = """
function() {
if ("division" in this) {
emit(this.division, {count: this.email_count})
}
}
"""
m = Code(mapcode)
try:
results = Target.objects().map_reduce(m, r, 'inline')
for result in results:
div = Division.objects(division__iexact=result.key).first()
if not div:
div = Division()
div.division = result.key
div.email_count = result.value['count']
div.save()
except:
raise
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def get_campaign_targets(campaign,user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
# Get addresses from the 'to' field of emails attributed to this campaign
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
addresses[to.strip().lower()] = 1 # add the way it should be
addresses[to] = 1 # also add the way it is in the Email
# Get addresses of Targets attributed to this campaign
targets = Target.objects(campaign__name=campaign).only('email_address')
for target in targets:
addresses[target.email_address] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def upload_attach(request, email_id):
"""
Upload an attachment for an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to upload attachment for.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
form = UploadFileForm(request.user, request.POST, request.FILES)
if form.is_valid():
cleaned_data = form.cleaned_data
analyst = request.user.username
users_sources = user_sources(analyst)
method = cleaned_data['method'] or "Add to Email"
bucket_list = cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
email_addr = None
if request.POST.get('email'):
email_addr = request.user.email
email = Email.objects(id=email_id, source__name__in=users_sources).first()
if not email:
return render_to_response('file_upload_response.html',
{'response': json.dumps({'success': False,
'message': "Could not find email."})},
RequestContext(request))
result = create_email_attachment(email,
cleaned_data,
analyst,
cleaned_data['source'],
method,
cleaned_data['reference'],
cleaned_data['campaign'],
cleaned_data['confidence'],
bucket_list,
ticket,
request.FILES.get('filedata',None),
request.POST.get('filename', None),
request.POST.get('md5', None),
email_addr,
cleaned_data['inherit_sources'])
# If successful, tell the browser to redirect back to this email.
if result['success']:
result['redirect_url'] = reverse('crits.emails.views.email_detail', args=[email_id])
return render_to_response('file_upload_response.html',
{'response': json.dumps(result)},
RequestContext(request))
else:
form.fields['related_md5'].widget = forms.HiddenInput() #hide field so it doesn't reappear
return render_to_response('file_upload_response.html',
{'response': json.dumps({'success': False,
'form': form.as_table()})},
RequestContext(request))
else:
return HttpResponseRedirect(reverse('crits.emails.views.email_detail',
args=[email_id]))
def execute_anb_campaign(cid, sources):
data = {'emails': '', 'samples': '', 'objects': ''}
email_list = Email.objects(campaign__name=cid, source__name__in=sources)
if not email_list:
return data
md5_list = []
for email in email_list:
md5_list = get_sample_rels(email.relationships, str(email.id), sources)
email.sanitize_sources(sources=sources)
data['emails'] += "%s,%s,%s,%s,%s,%s,%s,%s\r\n" % (
email.id,
email.isodate,
email.sender,
email.subject,
email.x_originating_ip,
email.x_mailer,
email.source[0].name,
email.campaign[0].name)
for m in md5_list:
data['samples'] += "%s,%s,%s,%s,%s\r\n" % (
m['email_id'],
m['md5'],
m['mimetype'],
m['backdoor'],
m['filename'])
for o in m.get('objects', []):
data['objects'] += "%s,%s\r\n" % (m['md5'], o)
return data
def execute_anb_campaign(cid, sources):
data = {'emails': '', 'samples': '', 'objects': ''}
email_list = Email.objects(campaign__name=cid, source__name__in=sources)
if not email_list:
return data
md5_list = []
for email in email_list:
md5_list = get_sample_rels(email.relationships, str(email.id), sources)
email.sanitize_sources(sources=sources)
data['emails'] += "%s,%s,%s,%s,%s,%s,%s,%s\r\n" % (
email.id, email.isodate, email.sender, email.subject,
email.x_originating_ip, email.x_mailer, email.source[0].name,
email.campaign[0].name)
for m in md5_list:
data['samples'] += "%s,%s,%s,%s,%s\r\n" % (
m['email_id'], m['md5'], m['mimetype'], m['backdoor'],
m['filename'])
for o in m.get('objects', []):
data['objects'] += "%s,%s\r\n" % (m['md5'], o)
return data
def indicator_from_header_field(request, email_id):
"""
Create an indicator from a header field. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to get the header from.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
if 'type' in request.POST:
header_field = request.POST.get('field')
header_type = request.POST.get('type')
analyst = request.user.username
sources = user_sources(analyst)
email = Email.objects(id=email_id,
source__name__in=sources).first()
if not email:
result = {'success': False, 'message': "Could not find email."}
else:
result = create_indicator_from_header_field(
email, header_field, header_type, analyst, request)
else:
result = {'success': False, 'message': "Type is a required value."}
return HttpResponse(json.dumps(result), mimetype="application/json")
else:
return render_to_response('error.html',
{'error': "Expected AJAX POST"},
RequestContext(request))
def indicator_from_header_field(request, email_id):
"""
Create an indicator from a header field. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to get the header from.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
if "type" in request.POST:
header_field = request.POST["type"]
analyst = request.user.username
sources = user_sources(analyst)
email = Email.objects(id=email_id, source__name__in=sources).first()
if not email:
result = {"success": False, "message": "Could not find email."}
else:
if header_field in ("from_address, sender, reply_to"):
ind_type = "Address - e-mail"
elif header_field in ("originating_ip", "x_originating_ip"):
ind_type = "Address - ipv4-addr"
else:
ind_type = "String"
result = create_indicator_from_header_field(email, header_field, ind_type, analyst, request)
else:
result = {"success": False, "message": "Type is a required value."}
return HttpResponse(json.dumps(result), mimetype="application/json")
else:
return render_to_response("error.html", {"error": "Expected AJAX POST"}, RequestContext(request))
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
# Get addresses from the 'to' field of emails attributed to this campaign
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
addresses[to.strip().lower()] = 1 # add the way it should be
addresses[to] = 1 # also add the way it is in the Email
# Get addresses of Targets attributed to this campaign
targets = Target.objects(campaign__name=campaign).only('email_address')
for target in targets:
addresses[target.email_address] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def upload_attach(request, email_id):
"""
Upload an attachment for an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to upload attachment for.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
analyst = request.user.username
sources = user_sources(analyst)
email = Email.objects(id=email_id, source__name__in=sources).first()
if not email:
error = "Could not find email."
return render_to_response("error.html", {"error": error},
RequestContext(request))
if request.method == 'POST':
form = EmailAttachForm(request.user.username, request.POST,
request.FILES)
if form.is_valid():
cleaned_data = form.cleaned_data
reference = cleaned_data['source_reference']
campaign = cleaned_data['campaign']
confidence = cleaned_data['confidence']
source = cleaned_data['source']
bucket_list = cleaned_data.get(
form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if request.FILES or 'filename' in request.POST and 'md5' in request.POST:
result = create_email_attachment(
email,
cleaned_data,
reference,
source,
analyst,
campaign=campaign,
confidence=confidence,
bucket_list=bucket_list,
ticket=ticket,
files=request.FILES.get('filedata', None),
filename=request.POST.get('filename', None),
md5=request.POST.get('md5', None))
if not result['success']:
return render_to_response("error.html",
{"error": result['message']},
RequestContext(request))
return HttpResponseRedirect(
reverse('crits.emails.views.email_detail', args=[email_id]))
else:
return render_to_response("error.html",
{"error": '%s' % form.errors},
RequestContext(request))
else:
return HttpResponseRedirect(
reverse('crits.emails.views.email_detail', args=[email_id]))
def upload_attach(request, email_id):
"""
Upload an attachment for an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to upload attachment for.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
analyst = request.user.username
sources = user_sources(analyst)
email = Email.objects(id=email_id, source__name__in=sources).first()
if not email:
error = "Could not find email."
return render_to_response("error.html",
{"error": error},
RequestContext(request))
if request.method == 'POST':
form = EmailAttachForm(request.user.username,
request.POST,
request.FILES)
if form.is_valid():
cleaned_data = form.cleaned_data
reference = cleaned_data['source_reference']
campaign = cleaned_data['campaign']
confidence = cleaned_data['confidence']
source = cleaned_data['source']
bucket_list = cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if request.FILES or 'filename' in request.POST and 'md5' in request.POST:
result = create_email_attachment(email,
cleaned_data,
reference,
source,
analyst,
campaign=campaign,
confidence=confidence,
bucket_list=bucket_list,
ticket=ticket,
files=request.FILES.get('filedata',None),
filename=request.POST.get('filename', None),
md5=request.POST.get('md5', None))
if not result['success']:
return render_to_response("error.html",
{"error": result['message'] },
RequestContext(request))
return HttpResponseRedirect(reverse('crits.emails.views.email_detail',
args=[email_id]))
else:
return render_to_response("error.html",
{"error": '%s' % form.errors },
RequestContext(request))
else:
return HttpResponseRedirect(reverse('crits.emails.views.email_detail',
args=[email_id]))
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def handle(self, *args, **options):
"""
Script Execution.
"""
self.is_delete = options.get('is_delete')
mapcode = """
function () {
try {
this.to.forEach(function(z) {
emit(z.toLowerCase(), {count: 1});
});
} catch(err) {}
}
"""
reducecode = """
function(k,v) {
var count = 0;
v.forEach(function(v) {
count += v["count"];
});
return {count: count};
}
"""
m = Code(mapcode)
r = Code(reducecode)
results = Email.objects(to__exists=True).map_reduce(m, r, 'inline')
for result in results:
try:
targets = Target.objects(email_address__iexact=result.key)
targ_dup_count = targets.count()
if targ_dup_count > 1:
print str(result.key) + " [" + str(targ_dup_count) + "]"
for target in targets:
print target.to_json()
if self.is_delete:
delete_up_to = targets.count() - 1
for target in targets[:delete_up_to]:
print "Deleting target: " + str(target.id)
target.delete()
except Exception, e:
print e
pass
def handle(self, *args, **options):
"""
Script Execution.
"""
self.is_delete = options.get('is_delete')
mapcode = """
function () {
try {
this.to.forEach(function(z) {
emit(z.toLowerCase(), {count: 1});
});
} catch(err) {}
}
"""
reducecode = """
function(k,v) {
var count = 0;
v.forEach(function(v) {
count += v["count"];
});
return {count: count};
}
"""
m = Code(mapcode)
r = Code(reducecode)
results = Email.objects(to__exists=True).map_reduce(m, r, 'inline')
for result in results:
try:
targets = Target.objects(email_address__iexact=result.key)
targ_dup_count = targets.count()
if targ_dup_count > 1:
print str(result.key) + " [" + str(targ_dup_count) + "]"
for target in targets:
print target.to_json()
if self.is_delete:
delete_up_to = targets.count() - 1
for target in targets[:delete_up_to]:
print "Deleting target: " + str(target.id)
target.delete()
except Exception, e:
print e
pass
def email_del(request, email_id):
"""
Delete an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to delete.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
email = Email.objects(id=email_id).first()
if email:
email.delete(username=request.user.username)
return HttpResponseRedirect(reverse("crits.emails.views.emails_listing"))
else:
return render_to_response("error.html", {"error": "Could not delete email."}, RequestContext(request))
def email_del(request, email_id):
"""
Delete an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to delete.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
email = Email.objects(id=email_id).first()
if email:
email.delete(username=request.user.username)
return HttpResponseRedirect(reverse('crits.emails.views.emails_listing'))
else:
return render_to_response('error.html',
{'error': "Could not delete email."},
RequestContext(request))
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
emails = Email.objects(source__name__in=sourcefilt, campaign__name=campaign).only("to")
addresses = {}
for email in emails:
for to in email["to"]:
addresses[to] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def indicator_from_header_field(request, email_id):
"""
Create an indicator from a header field. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to get the header from.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
if 'type' in request.POST:
header_field = request.POST.get('field')
header_type = request.POST.get('type')
analyst = request.user.username
sources = user_sources(analyst)
email = Email.objects(id=email_id,
source__name__in=sources).first()
if not email:
result = {
'success': False,
'message': "Could not find email."
}
else:
result = create_indicator_from_header_field(email,
header_field,
header_type,
analyst,
request)
else:
result = {
'success': False,
'message': "Type is a required value."
}
return HttpResponse(json.dumps(result),
content_type="application/json")
else:
return render_to_response('error.html',
{'error': "Expected AJAX POST"},
RequestContext(request))
def email_del(request, email_id):
"""
Delete an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to delete.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
user = request.user
email = Email.objects(id=email_id).first()
if not user.has_access_to(EmailACL.DELETE):
return render(request, 'error.html',
{'error':'User does not have permission to delete email.'})
if not email:
return render(request, 'error.html', {'error': "Could not delete email."})
email.delete(username=request.user.username)
return HttpResponseRedirect(reverse('crits-emails-views-emails_listing'))
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
addresses[to] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id, 'Campaign')
# subscription
subscription = {
'type': 'Campaign',
'id': campaign_detail.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Campaign',
campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {'comments': campaign_detail.get_comments(),
'url_key': campaign_name}
#screenshots
screenshots = campaign_detail.get_screenshots(analyst)
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(source__name__in=sources,
__raw__=formatted_query).count()
# Item counts for targets
emails = Email.objects(source__name__in=sources, __raw__=formatted_query)
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
# analysis results
service_results = campaign_detail.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"screenshots": screenshots,
'service_results': service_results,
"ttp_form": ttp_form}
return template, args
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id,
'Campaign')
# subscription
subscription = {
'type':
'Campaign',
'id':
campaign_detail.id,
'subscribed':
is_user_subscribed("%s" % analyst, 'Campaign', campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {
'comments': campaign_detail.get_comments(),
'url_key': campaign_name
}
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(
source__name__in=sources, __raw__=formatted_query).count()
# Item counts for targets
emails = Email.objects(source__name__in=sources, __raw__=formatted_query)
addresses = {}
for email in emails:
for to in email['to']:
addresses[to] = 1
uniq_addrs = addresses.keys()
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
args = {
'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"ttp_form": ttp_form
}
return template, args
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'Action':
return Action.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'Signature':
return Signature.objects(id=_id).first()
elif type_ == 'SignatureType':
return SignatureType.objects(id=_id).first()
elif type_ == 'SignatureDependency':
return SignatureDependency.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def campaign_heatmap(request):
campaigns = Campaign.objects().only('name', 'aliases', 'locations')
events = Event.objects().only('title', 'created', 'locations', 'campaign')
emails = Email.objects().only('created', 'locations', 'campaign')
# list of countries in alphabetical order. set 0 for the amount of campaign
# associated with this country for the next step.
country_list = []
for c in campaigns:
if len(c.locations):
for l in c.locations:
if [l.location,0] not in country_list:
country_list.append([l.location,0])
country_list.sort()
# For those campaigns with no location assigned, have an Unknown location.
country_list.append(['Unknown', 0])
# list of campaigns in order of country, then alphabetical by name
campaign_list = []
# for each country we build a tmp list, find all campaigns for that country,
# sort the list, then append it to the campaign list. bump the count so we
# know how many columns to span.
for c in country_list:
tmp = []
for cam in campaigns:
if len(cam.locations):
for l in cam.locations:
if l.location == c[0]:
c[1] += 1
if cam.name not in tmp:
tmp.append(cam.name)
break
else:
# Assuming we are checking the Unknown location, if this
# campaign has no location assigned, add it to Unknown.
if c[0] == 'Unknown':
c[1] += 1
if cam.name not in tmp:
tmp.append(cam.name)
# If we haven't added a campaign to this location, show "No Campaigns".
# This also prevents a left-shift in the counting and header rows.
if len(tmp) == 0:
tmp.append("No Campaigns")
tmp.sort()
campaign_list += tmp
# list of the months going back in history and the activity of each campaign
# during that month
month_list = []
# for each campaign, find associated events and emails. For each event and
# email, use the created date to put it into the appropriate list.
month_d = {}
idx = 0
# this is a default row in the heatmap with all values set to 0.
pad_list = [0 for _ in range(len(campaign_list))]
for c in campaign_list:
build_month_d(pad_list, month_d, c, idx, events)
build_month_d(pad_list, month_d, c, idx, emails)
idx += 1
# sort the months in reverse order for descending display.
for key in sorted(month_d, reverse=True):
month_list.append([key, month_d[key]])
final_data = {
'country_list': country_list,
'campaign_list': campaign_list,
'month_list': month_list,
}
return final_data
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
'Indicator', 'Screenshot'] and
not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def find_emails(self, username):
sources = user_sources(username)
emails = Email.objects(to=self.email_address, source__name__in=sources)
return emails
def to_stix(self, username=None):
"""
Converts a CRITs event to a STIX document.
The resulting document includes all related emails, samples, and
indicators converted to CybOX Observable objects.
Returns the STIX document and releasability constraints.
(NOTE: the following statement is untrue until the
releasability checking is finished, which includes setting
releasability on all CRITs objects.)
Raises UnreleasableEventError if the releasability on the
relationships and the event do not share any common releasability
sources.
"""
from crits.emails.email import Email
from crits.samples.sample import Sample
from crits.indicators.indicator import Indicator
from cybox.common import Time, ToolInformationList, ToolInformation
from cybox.core import Observables
from stix.common import StructuredText
from stix.core import STIXPackage, STIXHeader
from stix.common import InformationSource
from stix.common.identity import Identity
stix_indicators = []
stix_observables = []
final_objects = []
# create a list of sources to send as part of the results.
# list should be limited to the sources this user is allowed to use.
# this list should be used along with the list of objects to set the
# appropriate source's 'released' key to True for each object.
final_sources = []
user_source_list = user_sources(username)
for f in self.releasability:
if f.name in user_source_list:
final_sources.append(f.name)
final_sources = set(final_sources)
# TODO: eventually we can use class_from_id instead of the if block
# but only once we support all CRITs types.
for r in self.relationships:
obj = None
if r.rel_type == Email._meta['crits_type']:
obj = Email.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_cybox()
stix_observables.append(ind[0])
elif r.rel_type == Sample._meta['crits_type']:
obj = Sample.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_cybox()
for i in ind:
stix_observables.append(i)
elif r.rel_type == Indicator._meta['crits_type']:
#NOTE: Currently this will raise an exception if there
# are multiple indicators with the same value.
# Should be fixed automatically once we transition
# indicators to be related based on ObjectId rather
# than value.
obj = Indicator.objects(id=r.object_id,
source__name__in=user_source_list).first()
if obj:
ind, releas = obj.to_stix_indicator()
stix_indicators.append(ind)
else:
continue
#Create a releasability list that is the intersection of
# each related item's releasability with the event's
# releasability. If the resulting set is empty, raise exception
#TODO: Set releasability on all objects so that we actually
# get results here instead of always raising an exception.
if obj:
releas_sources = set([rel.name for rel in releas])
final_sources = final_sources.intersection(releas_sources)
#TODO: uncomment the following lines when objects have
# releasability set.
#if not final_sources:
# raise UnreleasableEventError(r.value)
# add to the final_objects list to send as part of the results
final_objects.append(obj)
tool_list = ToolInformationList()
tool = ToolInformation("CRITs", "MITRE")
tool.version = settings.CRITS_VERSION
tool_list.append(tool)
i_s = InformationSource(
time=Time(produced_time= datetime.datetime.now()),
identity = Identity(name=settings.COMPANY_NAME),
tools = tool_list
)
description = StructuredText(value=self.description)
header = STIXHeader(information_source=i_s,
description=description,
package_intent=self.event_type,
title=self.title)
return (STIXPackage(indicators=stix_indicators,
observables=Observables(stix_observables),
stix_header=header,
id_=self.event_id),
final_sources,
final_objects)
def find_emails(self, username):
sources = user_sources(username)
emails = Email.objects(to=self.email_address,
source__name__in=sources)
return emails
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in [
'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator',
'Screenshot'
] and not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(message_id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Signature':
return Signature.objects(md5=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def __parse_object(self, obs_obj):
"""
Parse an observable object.
:param obs_obj: The observable object to parse.
:type obs_obj: CybOX object type.
"""
properties = obs_obj.properties
type_ = properties._XSI_TYPE
#would isinstance be preferable?
#elif isinstance(defined_obj,
# cybox.objects.email_message_object.EmailMessage):
#XXX: Need to check the database for an existing Sample or Indicator
# and handle accordingly, or risk blowing it away!!!!
if type_ == 'FileObjectType':
sample = Sample.from_cybox(properties, [self.source])
md5_ = sample.md5
# do we already have this sample?
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'FileObjectType':
#print "Only File found in SA"
return
elif saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Found matching Artifact in SA"
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving File to SA"
self.saved_artifacts[md5_] = (properties, None)
elif type_ == 'EmailMessageObjectType':
# we assume all emails coming in from TAXII are new emails.
# there is no way to guarantee we found a dupe in the db.
email = Email.from_cybox(properties, [self.source])
email.save(username=self.source_instance.analyst)
self.emails.append(('Email', str(email.id)))
elif type_ in ['URIObjectType', 'AddressObjectType']:
indicator = Indicator.from_cybox(properties, [self.source])
ind_type = indicator.ind_type
value = indicator.value
db_indicator = Indicator.objects(
Q(ind_type=ind_type) & Q(value=value)).first()
if db_indicator:
# flat out replacing cybox indicator object with one from db.
# we add the source to track we got a copy from TAXII.
indicator = db_indicator
indicator.add_source(self.source)
indicator.save(username=self.source_instance.analyst)
self.indicators.append(('Indicator', str(indicator.id)))
elif type_ == 'ArtifactObjectType':
# XXX: Check properties.type_ to see if it is TYPE_FILE,
# TYPE_MEMORY, from CybOX definitions. This isn't implemented
# yet in Greg's code. Just parse the file blindly for now.
#if properties.type_ == 'File':
# sample = Sample.from_cybox(properties, [self.source])
#else:
# print "XXX: got unknown artifact type %s" % properties.type_
data = base64.b64decode(properties.data)
md5_ = md5(data).hexdigest()
#print "Found Artifact"
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Only Artifact found in SA"
return
elif saved_obj._XSI_TYPE == 'FileObjectType':
#print "Found matching File in SA"
sample = Sample.from_cybox(saved_obj, [self.source])
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving Artifact to SA"
self.saved_artifacts[md5_] = (properties, data)
def upload_attach(request, email_id):
"""
Upload an attachment for an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to upload attachment for.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
redirect = reverse('crits-emails-views-email_detail', args=[email_id])
user = request.user
if request.method != 'POST':
return HttpResponseRedirect(redirect)
file_form = UploadFileForm(request.user, request.POST, request.FILES)
json_reply = {'success': False}
if not file_form.is_valid():
file_form.fields['related_md5_event'].widget = forms.HiddenInput() #hide field so it doesn't reappear
json_reply['form'] = file_form.as_table()
return render(request, 'file_upload_response.html', {'response': json.dumps(json_reply)})
if not user.has_access_to(EmailACL.ADD_ATTACHMENT):
json_reply['message'] = "User does not have permission to upload attachment."
return render(request, 'file_upload_response.html',
{'response': json.dumps(json_reply)})
analyst = request.user.username
users_sources = user_sources(analyst)
method = file_form.cleaned_data['method'] or "Add to Email"
bucket_list = file_form.cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = file_form.cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
email_addr = None
if request.POST.get('email'):
email_addr = request.user.email
email = Email.objects(id=email_id,
source__name__in=users_sources).first()
if not email:
json_reply['message'] = "Could not find email."
return render(request, 'file_upload_response.html', {'response': json.dumps(json_reply)})
result = create_email_attachment(email,
file_form,
analyst,
file_form.cleaned_data['source'],
method,
file_form.cleaned_data['reference'],
file_form.cleaned_data['campaign'],
file_form.cleaned_data['confidence'],
bucket_list,
ticket,
request.FILES.get('filedata'),
request.POST.get('filename'),
request.POST.get('md5'),
email_addr,
file_form.cleaned_data['inherit_sources'])
# If successful, tell the browser to redirect back to this email.
if result['success']:
result['redirect_url'] = redirect
return render(request, 'file_upload_response.html', {'response': json.dumps(result)})
def campaign_heatmap(request):
campaigns = Campaign.objects().only('name', 'aliases', 'locations')
events = Event.objects().only('title', 'created', 'locations', 'campaign')
emails = Email.objects().only('created', 'locations', 'campaign')
# list of countries in alphabetical order. set 0 for the amount of campaign
# associated with this country for the next step.
country_list = []
for c in campaigns:
if len(c.locations):
for l in c.locations:
if [l.location, 0] not in country_list:
country_list.append([l.location, 0])
country_list.sort()
# For those campaigns with no location assigned, have an Unknown location.
country_list.append(['Unknown', 0])
# list of campaigns in order of country, then alphabetical by name
campaign_list = []
# for each country we build a tmp list, find all campaigns for that country,
# sort the list, then append it to the campaign list. bump the count so we
# know how many columns to span.
for c in country_list:
tmp = []
for cam in campaigns:
if len(cam.locations):
for l in cam.locations:
if l.location == c[0]:
c[1] += 1
if cam.name not in tmp:
tmp.append(cam.name)
break
else:
# Assuming we are checking the Unknown location, if this
# campaign has no location assigned, add it to Unknown.
if c[0] == 'Unknown':
c[1] += 1
if cam.name not in tmp:
tmp.append(cam.name)
# If we haven't added a campaign to this location, show "No Campaigns".
# This also prevents a left-shift in the counting and header rows.
if len(tmp) == 0:
tmp.append("No Campaigns")
tmp.sort()
campaign_list += tmp
# list of the months going back in history and the activity of each campaign
# during that month
month_list = []
# for each campaign, find associated events and emails. For each event and
# email, use the created date to put it into the appropriate list.
month_d = {}
idx = 0
# this is a default row in the heatmap with all values set to 0.
pad_list = [0 for _ in range(len(campaign_list))]
for c in campaign_list:
build_month_d(pad_list, month_d, c, idx, events)
build_month_d(pad_list, month_d, c, idx, emails)
idx += 1
# sort the months in reverse order for descending display.
for key in sorted(month_d, reverse=True):
month_list.append([key, month_d[key]])
final_data = {
'country_list': country_list,
'campaign_list': campaign_list,
'month_list': month_list,
}
return final_data
def __parse_object(self, obs_obj):
"""
Parse an observable object.
:param obs_obj: The observable object to parse.
:type obs_obj: CybOX object type.
"""
properties = obs_obj.properties
type_ = properties._XSI_TYPE
#would isinstance be preferable?
#elif isinstance(defined_obj,
# cybox.objects.email_message_object.EmailMessage):
#XXX: Need to check the database for an existing Sample or Indicator
# and handle accordingly, or risk blowing it away!!!!
if type_ == 'FileObjectType':
sample = Sample.from_cybox(properties, [self.source])
md5_ = sample.md5
# do we already have this sample?
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'FileObjectType':
#print "Only File found in SA"
return
elif saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Found matching Artifact in SA"
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving File to SA"
self.saved_artifacts[md5_] = (properties, None)
elif type_ == 'EmailMessageObjectType':
# we assume all emails coming in from TAXII are new emails.
# there is no way to guarantee we found a dupe in the db.
email = Email.from_cybox(properties, [self.source])
email.save(username=self.source_instance.analyst)
self.emails.append(('Email', str(email.id)))
elif type_ in ['URIObjectType', 'AddressObjectType']:
indicator = Indicator.from_cybox(properties, [self.source])
ind_type = indicator.ind_type
value = indicator.value
db_indicator = Indicator.objects(Q(ind_type=ind_type) & Q(value=value)).first()
if db_indicator:
# flat out replacing cybox indicator object with one from db.
# we add the source to track we got a copy from TAXII.
indicator = db_indicator
indicator.add_source(self.source)
indicator.save(username=self.source_instance.analyst)
self.indicators.append(('Indicator', str(indicator.id)))
elif type_ == 'ArtifactObjectType':
# XXX: Check properties.type_ to see if it is TYPE_FILE,
# TYPE_MEMORY, from CybOX definitions. This isn't implemented
# yet in Greg's code. Just parse the file blindly for now.
#if properties.type_ == 'File':
# sample = Sample.from_cybox(properties, [self.source])
#else:
# print "XXX: got unknown artifact type %s" % properties.type_
data = base64.b64decode(properties.data)
md5_ = md5(data).hexdigest()
#print "Found Artifact"
if md5_ in self.saved_artifacts:
(saved_obj, data) = self.saved_artifacts[md5_]
if saved_obj._XSI_TYPE == 'ArtifactObjectType':
#print "Only Artifact found in SA"
return
elif saved_obj._XSI_TYPE == 'FileObjectType':
#print "Found matching File in SA"
sample = Sample.from_cybox(saved_obj, [self.source])
db_sample = Sample.objects(md5=md5_).first()
if db_sample:
# flat out replacing cybox sample object with one from db.
# we add the source to track we got a copy from TAXII.
# if we have a metadata only doc, the add_file_data below
# will generate metadata for us.
sample = db_sample
sample.add_source(self.source)
sample.add_file_data(data)
sample.save(username=self.source_instance.analyst)
self.samples.append(('Sample', sample.md5))
del self.saved_artifacts[md5_]
else:
#print "Saving Artifact to SA"
self.saved_artifacts[md5_] = (properties, data)
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# Quick fail
if not _id or not type_:
return None
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode("utf8")):
return None
if type_ == "Actor":
return Actor.objects(id=_id).first()
elif type_ == "Backdoor":
return Backdoor.objects(id=_id).first()
elif type_ == "ActorThreatIdentifier":
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == "Campaign":
return Campaign.objects(id=_id).first()
elif type_ == "Certificate":
return Certificate.objects(id=_id).first()
elif type_ == "Comment":
return Comment.objects(id=_id).first()
elif type_ == "Domain":
return Domain.objects(id=_id).first()
elif type_ == "Email":
return Email.objects(id=_id).first()
elif type_ == "Event":
return Event.objects(id=_id).first()
elif type_ == "Exploit":
return Exploit.objects(id=_id).first()
elif type_ == "Indicator":
return Indicator.objects(id=_id).first()
elif type_ == "Action":
return Action.objects(id=_id).first()
elif type_ == "IP":
return IP.objects(id=_id).first()
elif type_ == "PCAP":
return PCAP.objects(id=_id).first()
elif type_ == "RawData":
return RawData.objects(id=_id).first()
elif type_ == "RawDataType":
return RawDataType.objects(id=_id).first()
elif type_ == "Sample":
return Sample.objects(id=_id).first()
elif type_ == "Signature":
return Signature.objects(id=_id).first()
elif type_ == "SignatureType":
return SignatureType.objects(id=_id).first()
elif type_ == "SignatureDependency":
return SignatureDependency.objects(id=_id).first()
elif type_ == "SourceAccess":
return SourceAccess.objects(id=_id).first()
elif type_ == "Screenshot":
return Screenshot.objects(id=_id).first()
elif type_ == "Target":
return Target.objects(id=_id).first()
elif type_ == "UserRole":
return UserRole.objects(id=_id).first()
else:
return None
