def snugglefish_search(indexes, search, user):
"""Execute search of selected index with the given string."""
# Return a dictionary where the key is the index name and the
# value a dictionary with status and a list of potential matches.
ret = {}
# If there are no sources, return early.
sources = user_sources(user)
if not sources:
return ret
for idx in indexes:
ret[idx] = {
'success': True,
'reason': '',
'files': []
}
sngindex = SnuggleIndex.objects(name=idx).first()
if not sngindex:
ret[idx]['reason'] = "Index not found in database."
ret[idx]['success'] = False
continue
snuggle = pysnugglefish.init(str(sngindex.directory + "/" + idx))
try:
tmp = snuggle.search(search)
for res in tmp:
if Sample.objects(md5=res, source__name__in=sources).count() > 0:
ret[idx]['files'].append(res)
except Exception, e:
ret[idx]['reason'] = "Error: %s" % e
ret[idx]['success'] = False
def activity_update(id_, activity, user=None, **kwargs):
"""
Update activity for an Indicator.
:param id_: The ObjectId of the indicator to update.
:type id_: str
:param activity: The activity information.
:type activity: dict
:param user: The user updating the activity.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"object" (dict) if successful.
"""
sources = user_sources(user)
indicator = Indicator.objects(id=id_, source__name__in=sources).first()
if not indicator:
return {"success": False, "message": "Could not find Indicator"}
try:
activity = datetime_parser(activity)
activity["analyst"] = user
indicator.edit_activity(
activity["analyst"], activity["start_date"], activity["end_date"], activity["description"], activity["date"]
)
indicator.save(username=user)
return {"success": True, "object": activity}
except ValidationError, e:
return {"success": False, "message": e}
def add_results(object_type, object_id, analysis_id, result, type_, subtype,
analyst):
"""
Add multiple results to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param result: The list of result to append.
:type result: list of str
:param type_: The list of result types.
:type type_: list of str
:param subtype: The list of result subtypes.
:type subtype: list of str
:param analyst: The user updating the results.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
res = {'success': False}
if not object_type or not object_id or not analysis_id:
res['message'] = "Must supply object id/type and analysis id."
return res
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
res['message'] = "Could not find object to add results to."
return res
if not(result and type_ and subtype):
res['message'] = "Need a result, type, and subtype to add a result."
return res
if not(len(result) == len(type_) == len(subtype)):
res['message'] = "result, type, and subtype need to be the same length."
return res
# Update analysis results
final_list = []
for key, r in enumerate(result):
final = {}
final['subtype'] = subtype[key]
final['result'] = r
tmp = ast.literal_eval(type_[key])
for k in tmp:
final[k] = tmp[k]
final_list.append(final)
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(push_all__results=final_list)
res['success'] = True
return res
def activity_update(indicator_id, activity):
"""
Update activity for an Indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param activity: The activity information.
:type activity: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"object" (dict) if successful.
"""
sources = user_sources(activity['analyst'])
indicator = Indicator.objects(id=indicator_id,
source__name__in=sources).first()
if not indicator:
return {'success': False,
'message': 'Could not find Indicator'}
try:
indicator.edit_activity(activity['analyst'],
activity['start_date'],
activity['end_date'],
activity['description'],
activity['date'])
indicator.save(username=activity['analyst'])
return {'success': True, 'object': activity}
except ValidationError, e:
return {'success': False, 'message': e}
def get_diffie_config(analyst, type_, id_, data=None):
"""
Return DiffieConfigForm for diffie service.
Must make sure user has source access to requested type and id.
:param analyst: The username.
:type analyst: str
:param type_: CRITs type.
:type type_: str
:param id_: CRITs ID.
:type id_: str
:param data: Dictionary with submitted data.
:type data: dict
:returns: DiffieConfigForm
"""
results = {'success': False}
sources = user_sources(analyst)
klass = class_from_type(type_)
obj = klass.objects(source__name__in=sources, id=id_).first()
if not obj:
results['message'] = "Either no data exists for this object or you do not have permission to view it."
return results
results['success'] = True
results['form'] = forms.DiffieConfigForm(type_=type_, id_=id_, data=data)
return results
def indicator_from_header_field(request, email_id):
"""
Create an indicator from a header field. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to get the header from.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
if "type" in request.POST:
header_field = request.POST["type"]
analyst = request.user.username
sources = user_sources(analyst)
email = Email.objects(id=email_id, source__name__in=sources).first()
if not email:
result = {"success": False, "message": "Could not find email."}
else:
if header_field in ("from_address, sender, reply_to"):
ind_type = "Address - e-mail"
elif header_field in ("originating_ip", "x_originating_ip"):
ind_type = "Address - ipv4-addr"
else:
ind_type = "String"
result = create_indicator_from_header_field(email, header_field, ind_type, analyst, request)
else:
result = {"success": False, "message": "Type is a required value."}
return HttpResponse(json.dumps(result), mimetype="application/json")
else:
return render_to_response("error.html", {"error": "Expected AJAX POST"}, RequestContext(request))
def set_backdoor_version(id_, version, user, **kwargs):
"""
Set a Backdoor name.
:param id_: Backdoor ObjectId.
:type id_: str
:param version: The new version.
:type version: str
:param user: The user updating the version.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(user)
backdoor = Backdoor.objects(id=id_, source__name__in=sources).first()
if not backdoor:
return {'success': False, 'message': "Could not find backdoor"}
# Make sure we don't have a backdoor with that name and new version yet.
existing_backdoor = Backdoor.objects(name=backdoor.name,
version=version,
source__name__in=sources).first()
if existing_backdoor:
# Be sure to return the same error message so backdoors can not be
# enumerated via this method.
return {'success': False, 'message': "Could not find backdoor"}
backdoor.version = version.strip()
backdoor.save(username=user)
return {'success': True}
def upload_attach(request, email_id):
"""
Upload an attachment for an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to upload attachment for.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
form = UploadFileForm(request.user, request.POST, request.FILES)
if form.is_valid():
cleaned_data = form.cleaned_data
analyst = request.user.username
users_sources = user_sources(analyst)
method = cleaned_data['method'] or "Add to Email"
bucket_list = cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
email_addr = None
if request.POST.get('email'):
email_addr = request.user.email
email = Email.objects(id=email_id, source__name__in=users_sources).first()
if not email:
return render_to_response('file_upload_response.html',
{'response': json.dumps({'success': False,
'message': "Could not find email."})},
RequestContext(request))
result = create_email_attachment(email,
cleaned_data,
analyst,
cleaned_data['source'],
method,
cleaned_data['reference'],
cleaned_data['campaign'],
cleaned_data['confidence'],
bucket_list,
ticket,
request.FILES.get('filedata',None),
request.POST.get('filename', None),
request.POST.get('md5', None),
email_addr,
cleaned_data['inherit_sources'])
# If successful, tell the browser to redirect back to this email.
if result['success']:
result['redirect_url'] = reverse('crits.emails.views.email_detail', args=[email_id])
return render_to_response('file_upload_response.html',
{'response': json.dumps(result)},
RequestContext(request))
else:
form.fields['related_md5'].widget = forms.HiddenInput() #hide field so it doesn't reappear
return render_to_response('file_upload_response.html',
{'response': json.dumps({'success': False,
'form': form.as_table()})},
RequestContext(request))
else:
return HttpResponseRedirect(reverse('crits.emails.views.email_detail',
args=[email_id]))
def get_aggregate_comments(atype, value, username, date=None):
"""
Generate a list of comments for the aggregate view.
:param atype: How to limit the comments ("bytag", "byuser", "bycomment").
:type atype: str
:param value: If limiting by atype, the value to limit by.
:type value: str
:param username: The user getting the comments.
:type username: str
:param date: The specific date to get comments for.
:type date: datetime.datetime
:returns: list of :class:`crits.comments.comment.Comment`
"""
results = None
if date:
end_date = date+datetime.timedelta(days=1)
query = {'date':{'$gte':date, '$lte':end_date}}
else:
query = {}
if atype == 'bytag':
query['tags'] = value
elif atype == 'byuser':
query['$or'] = [{'users':value}, {'analyst':value}]
elif atype == 'bycomment':
query['comment'] = {'$regex':value}
results = Comment.objects(__raw__=query)
sources = user_sources(username)
return get_user_allowed_comments(results, sources)
def remove_attribution(id_, identifier=None, user=None, **kwargs):
"""
Remove an attributed identifier.
:param id_: The ObjectId of the Actor.
:param identifier: The Actor Identifier ObjectId.
:type identifier: str
:param user: The user removing this attribution.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(user)
admin = is_admin(user)
actor = Actor.objects(id=id_,
source__name__in=sources).first()
if not actor:
return {'success': False,
'message': "Could not find actor"}
actor.remove_attribution(identifier)
actor.save(username=user)
actor.reload()
actor_identifiers = actor.generate_identifiers_list(user)
html = render_to_string('actor_identifiers_widget.html',
{'actor_identifiers': actor_identifiers,
'admin': admin,
'actor_id': str(actor.id)})
return {'success': True,
'message': html}
def set_identifier_confidence(id_, identifier=None, confidence="low",
user=None, **kwargs):
"""
Set the Identifier attribution confidence.
:param id_: The ObjectId of the Actor.
:param identifier: The Actor Identifier ObjectId.
:type identifier: str
:param confidence: The confidence level.
:type confidence: str
:param user: The user editing this identifier.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(user)
actor = Actor.objects(id=id_,
source__name__in=sources).first()
if not actor:
return {'success': False,
'message': "Could not find actor"}
actor.set_identifier_confidence(identifier, confidence)
actor.save(username=user)
return {'success': True}
def sanitize(self, username=None, sources=None, rels=None):
"""
Sanitize the source list down to only those a user has access to see.
This was sniped from core/crits_mongoengine.
:param username: The user requesting this data.
:type username: str
:param sources: A list of sources the user has access to.
:type sources: list
"""
if username and hasattr(self, 'source'):
length = len(self.source)
if not sources:
sources = user_sources(username)
# use slice to modify in place in case any code is referencing
# the source already will reflect the changes as well
self.source[:] = [s for s in self.source if s.name in sources]
# a bit of a hack but we add a poorly formatted source to the
# source list which has an instances length equal to the amount
# of sources that were sanitized out of the user's list.
# not tested but this has the added benefit of throwing a
# ValidationError if someone were to try and save() this.
new_length = len(self.source)
if length > new_length:
i_length = length - new_length
s = EmbeddedSource()
s.name = "Other"
s.instances = [0] * i_length
self.source.append(s)
def set_actor_name(id_, name, user, **kwargs):
"""
Set an Actor name.
:param id_: Actor ObjectId.
:type id_: str
:param name: The new name.
:type name: str
:param user: The user updating the name.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(user)
actor = Actor.objects(id=id_,
source__name__in=sources).first()
if not actor:
return {'success': False,
'message': "Could not find actor"}
actor.name = name.strip()
actor.save(username=user)
return {'success': True}
def action_add(indicator_id, action):
"""
Add an action to an indicator.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param action: The information about the action.
:type action: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"object" (dict) if successful.
"""
sources = user_sources(action['analyst'])
indicator = Indicator.objects(id=indicator_id,
source__name__in=sources).first()
if not indicator:
return {'success': False,
'message': 'Could not find Indicator'}
try:
indicator.add_action(action['action_type'],
action['active'],
action['analyst'],
action['begin_date'],
action['end_date'],
action['performed_date'],
action['reason'],
action['date'])
indicator.save(username=action['analyst'])
return {'success': True, 'object': action}
except ValidationError, e:
return {'success': False, 'message': e}
def set_actor_description(id_, description, analyst):
"""
Set an Actor description.
:param id_: Actor ObjectId.
:type id_: str
:param description: The new description.
:type description: str
:param analyst: The user updating the description.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(analyst)
actor = Actor.objects(id=id_,
source__name__in=sources).first()
if not actor:
return {'success': False,
'message': "Could not find actor"}
actor.description = description.strip()
actor.save(username=analyst)
return {'success': True}
def generate_identifiers_list(self, username=None):
"""
Create a list of dictionaries with Identifier information which can be
used for rendering in a template.
:returns: list
"""
result = []
if not username:
return result
sources = user_sources(username)
for i in self.identifiers:
it = ActorIdentifier.objects(id=i.identifier_id,
source__name__in=sources).first()
if it:
d = {}
d['analyst'] = i.analyst
d['confidence'] = i.confidence
d['id'] = i.identifier_id
d['type'] = it.identifier_type
d['name'] = it.name
d['date'] = i.date
result.append(d)
return result
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
# Get addresses from the 'to' field of emails attributed to this campaign
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
addresses[to.strip().lower()] = 1 # add the way it should be
addresses[to] = 1 # also add the way it is in the Email
# Get addresses of Targets attributed to this campaign
targets = Target.objects(campaign__name=campaign).only('email_address')
for target in targets:
addresses[target.email_address] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def get_campaign_targets(campaign,user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def get_screenshot(_id=None, tag=None, analyst=None, thumb=False):
"""
Get a screenshot.
:param _id: The ObjectId to lookup.
:type _id: str
:param tag: The tag to look for.
:type tag: str
:param analyst: The user looking up the screenshots.
:type analyst: str
:returns: screenshot
"""
if not analyst:
return None
sources = user_sources(analyst)
if _id:
screenshot = Screenshot.objects(id=_id,
source__name__in=sources).first()
if tag:
screenshot = Screenshot.objects(tags=tag,
source__name__in=sources).first()
if not screenshot:
return None
if thumb:
im = Image.open(screenshot.thumb)
else:
im = Image.open(screenshot.screenshot)
response = HttpResponse(content_type="image/png")
im.save(response, "PNG")
return response
def upload_attach(request, email_id):
"""
Upload an attachment for an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to upload attachment for.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
analyst = request.user.username
sources = user_sources(analyst)
email = Email.objects(id=email_id, source__name__in=sources).first()
if not email:
error = "Could not find email."
return render_to_response("error.html",
{"error": error},
RequestContext(request))
if request.method == 'POST':
form = EmailAttachForm(request.user.username,
request.POST,
request.FILES)
if form.is_valid():
cleaned_data = form.cleaned_data
reference = cleaned_data['source_reference']
campaign = cleaned_data['campaign']
confidence = cleaned_data['confidence']
source = cleaned_data['source']
bucket_list = cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if request.FILES or 'filename' in request.POST and 'md5' in request.POST:
result = create_email_attachment(email,
cleaned_data,
reference,
source,
analyst,
campaign=campaign,
confidence=confidence,
bucket_list=bucket_list,
ticket=ticket,
files=request.FILES.get('filedata',None),
filename=request.POST.get('filename', None),
md5=request.POST.get('md5', None))
if not result['success']:
return render_to_response("error.html",
{"error": result['message'] },
RequestContext(request))
return HttpResponseRedirect(reverse('crits.emails.views.email_detail',
args=[email_id]))
else:
return render_to_response("error.html",
{"error": '%s' % form.errors },
RequestContext(request))
else:
return HttpResponseRedirect(reverse('crits.emails.views.email_detail',
args=[email_id]))
def __init__(self, username, *args, **kwargs):
"""
Initialize the form.
"""
kwargs.setdefault('label_suffix', ':')
super(TAXIIPollForm, self).__init__(*args, **kwargs)
sc = get_config('taxii_service')
user_srcs = user_sources(username)
self.fields['feeds'].choices = get_taxii_feeds(user_srcs)
def add_result(object_type, object_id, analysis_id, result, type_, subtype,
analyst):
"""
Add a result to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param result: The result to append.
:type result: str
:param type_: The result type.
:type type_: str
:param subtype: The result subtype.
:type subtype: str
:param analyst: The user updating the results.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
res = {'success': False}
if not object_type or not object_id or not analysis_id:
res['message'] = "Must supply object id/type and analysis id."
return res
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
res['message'] = "Could not find object to add results to."
return res
found = False
c = 0
for a in obj.analysis:
if str(a.analysis_id) == analysis_id:
found = True
break
c += 1
if not found:
res['message'] = "Could not find an analysis task to update."
return res
if result and type_ and subtype:
final = {}
final['subtype'] = subtype
final['result'] = result
tmp = ast.literal_eval(type_)
for k in tmp:
final[k] = tmp[k]
klass.objects(id=object_id,
analysis__id=analysis_id).update_one(push__analysis__S__results=final)
else:
res['message'] = "Need a result, type, and subtype to add a result."
return res
res['success'] = True
return res
def update_dependency(type_, id_, dep, user, append=False, **kwargs):
"""
Change the dependencies needed for a signature
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: The ObjectId to search for.
:type id_: str
:param data_type_dependency: The new list of dependencies
:type data_type_dependency: list
:param user: The user setting the dependency.
:type user: str
:param append: Should be appended to dependency list?
:type append: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
# Have to unescape the submitted data. Use unescape() to escape
# < and friends. Use urllib2.unquote() to escape %3C and friends.
h = HTMLParser.HTMLParser()
data_type_dependency = h.unescape(dep)
try:
deps = data_type_dependency.split(',')
if append is False:
del obj.data_type_dependency[:]
for item in deps:
item = item.strip()
item = str(item)
if item:
add_new_signature_dependency(item, user)
obj.data_type_dependency.append(item)
obj.save(username=user)
return {'success': True, 'message': "Data type dependency set."}
except ValidationError, e:
return {'success': False, 'message': e}
def refresh_services(request, crits_type, identifier):
"""
Refresh the Analysis tab with the latest information.
"""
response = {}
# Verify user can see results.
sources = user_sources(request.user.username)
klass = class_from_type(crits_type)
if not klass:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), content_type="application/json")
if hasattr(klass, 'source'):
obj = klass.objects(id=identifier,source__name__in=sources).first()
else:
obj = klass.objects(id=identifier).first()
if not obj:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), content_type="application/json")
# Get analysis results.
results = AnalysisResult.objects(object_type=crits_type,
object_id=identifier)
relationship = {'type': crits_type,
'value': identifier}
subscription = {'type': crits_type,
'id': identifier}
service_list = get_supported_services(crits_type)
response['success'] = True
response['html'] = render_to_string("services_analysis_listing.html",
{'relationship': relationship,
'subscription': subscription,
'service_results': results,
'crits_type': crits_type,
'identifier': identifier,
'service_list': service_list},
RequestContext(request))
return HttpResponse(json.dumps(response), content_type="application/json")
def get_anb_data(request, ctype, cid):
result = { "success": "false", "message": "No data available." }
sources = user_sources("%s" % request.user)
if not sources:
return HttpResponse(json.dumps(result), mimetype="application/json")
data = handlers.execute_anb(ctype, cid, sources)
# If any of the values are not an empty string we have data.
for v in data.values():
if v != "":
result['success'] = "true"
result['message'] = data
break
return HttpResponse(json.dumps(result), mimetype="application/json")
def add_log(object_type, object_id, analysis_id, log_message, level, analyst):
"""
Add a log entry to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param log_message: The log entry to append.
:type log_message: dict
:param level: The log level.
:type level: str
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
results['message'] = "Could not find object to add log to."
return results
found = False
c = 0
for a in obj.analysis:
if str(a.analysis_id) == analysis_id:
found = True
break
c += 1
if not found:
results['message'] = "Could not find an analysis task to update."
return results
le = EmbeddedAnalysisResult.EmbeddedAnalysisResultLog()
le.message = log_message
le.level = level
le.datetime = str(datetime.datetime.now())
klass.objects(id=object_id,
analysis__id=analysis_id).update_one(push__analysis__S__log=le)
results['success'] = True
return results
def get_backdoor_names(user, **kwargs):
"""
Get a list of unique backdoor names.
:param user: The user requesting the names.
:returns: list of tuples
"""
sources = user_sources(user)
only = ('name', 'version')
bds = Backdoor.objects(source__name__in=sources).only(*only)
names = []
for b in bds:
if (b.name, b.version) not in names:
names.append((b.name, b.version))
return names
def finish_task(object_type, object_id, analysis_id, status, analyst):
"""
Finish a task by setting its status to "completed" and setting the finish
date.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param status: The status of the task.
:type status: str ("error", "completed")
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not status:
status = "completed"
if status not in ('error', 'completed'):
status = "completed"
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
params = {'id': object_id}
if hasattr(klass, 'source'):
params['source__name__in'] = user_sources(analyst)
obj = klass.objects(**params).first()
if not obj:
results['message'] = "Could not find object to add results to."
return results
# Update analysis log
date = str(datetime.datetime.now())
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(set__status=status,
set__finish_date=date)
results['success'] = True
return results
def attribute_actor_identifier(id_, identifier_type, identifier=None,
confidence="low", user=None, **kwargs):
"""
Attribute an Actor Identifier to an Actor in CRITs.
:param id_: The Actor ObjectId.
:type id_: str
:param identifier_type: The Actor Identifier Type.
:type identifier_type: str
:param identifier: The Actor Identifier.
:type identifier: str
:param user: The user attributing this identifier.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(user)
admin = is_admin(user)
actor = Actor.objects(id=id_,
source__name__in=sources).first()
if not actor:
return {'success': False,
'message': "Could not find actor"}
c = len(actor.identifiers)
actor.attribute_identifier(identifier_type, identifier, confidence, user)
actor.save(username=user)
actor.reload()
actor_identifiers = actor.generate_identifiers_list(user)
if len(actor.identifiers) <= c:
return {'success': False,
'message': "Invalid data submitted or identifier is already attributed."}
html = render_to_string('actor_identifiers_widget.html',
{'actor_identifiers': actor_identifiers,
'admin': admin,
'actor_id': str(actor.id)})
return {'success': True,
'message': html}
def get_screenshots_for_id(type_, _id, analyst, buckets=False):
"""
Get screenshots for a top-level object.
:param type_: The class type.
:type type_: str
:param _id: The ObjectId to lookup.
:type _id: str
:param analyst: The user looking up the screenshots.
:type analyst: str
:param buckets: Use buckets as tag lookups for screenshots.
:type buckets: boolean
:returns: list
"""
result = {'success': False}
sources = user_sources(analyst)
obj = class_from_id(type_, _id)
if not obj:
result['message'] = "No valid top-level object found."
return result
screenshots = Screenshot.objects(id__in=obj.screenshots,
source__name__in=sources)
bucket_shots = Screenshot.objects(tags__in=obj.bucket_list,
source__name__in=sources)
final_shots = []
for s in screenshots:
if s.screenshot and s.thumb and s not in final_shots:
final_shots.append(s)
for b in bucket_shots:
if b not in final_shots:
# since .bucket isn't supported, this will show up in the template
# under unsupported_attrs, which is ok.
b.bucket = True
final_shots.append(b)
result['success'] = True
result['screenshots'] = final_shots
return result
def finish_task(object_type, object_id, analysis_id, status, analyst):
"""
Finish a task by setting its status to "completed" and setting the finish
date.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param status: The status of the task.
:type status: str ("error", "completed")
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not status:
status = "completed"
if status not in ('error', 'completed'):
status = "completed"
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
results['message'] = "Could not find object to add log to."
return results
date = str(datetime.datetime.now())
klass.objects(id=object_id,
analysis__id=analysis_id).update_one(set__analysis__S__status=status,
set__analysis__S__finish_date=date)
results['success'] = True
return results
def update_signature_type(type_, id_, data_type, user, **kwargs):
"""
Update the Signature data type.
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: ObjectId of the Signature to update.
:type id_: str
:param data_type: The data type to set.
:type data_type: str
:param user: The user updating the data type.
:type user: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
signature = Signature.objects(id=id_).first()
data_type = SignatureType.objects(name=data_type).first()
if not data_type:
return None
else:
signature.data_type = data_type.name
try:
signature.save(username=user)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': str(e)}
def set_exploit_name(id_, name, user, **kwargs):
"""
Set a Exploit name.
:param id_: Exploit ObjectId.
:type id_: str
:param name: The new name.
:type name: str
:param user: The user updating the name.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(user)
exploit = Exploit.objects(id=id_, source__name__in=sources).first()
if not exploit:
return {'success': False, 'message': "Could not find exploit"}
exploit.name = name.strip()
exploit.save(username=user)
return {'success': True}
def __init__(self, username, item, *args, **kwargs):
"""
Initialize the form.
Populates form fields based on context object (item) and its related items.
The way the form fields are populated ensures that only STIXifyable / CybOXable
options are provided.
"""
super(TAXIIForm, self).__init__(*args, **kwargs)
sc = get_config('taxii_service')
# Avoid options that cause failure: set recipients to intersection of
# user's sources and the sources that have TAXII feeds configured
user_srcs = user_sources(username)
taxii_srcs = [crtfile.split(',')[0] for crtfile in sc['certfiles']]
self.fields['rcpts'].choices = [(n, n) for n in set(user_srcs).intersection(taxii_srcs)]
# populate all of the multi choice fields with valid options
# from the context CRITs object's related items.
for _type in get_supported_types(): # TODO the hardcoded args to collect_objects should be revisited
collected = collect_objects(item._meta['crits_type'], item.id, 1, 100, 100, [_type], user_srcs)
field = forms.MultipleChoiceField(required=False, label=_type)
field.choices = filter_and_format_choices(collected, item, _type)
self.fields[_type] = field
def set_actor_name(id_, name, user, **kwargs):
"""
Set an Actor name.
:param id_: Actor ObjectId.
:type id_: str
:param name: The new name.
:type name: str
:param user: The user updating the name.
:type user: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(user)
actor = Actor.objects(id=id_, source__name__in=sources).first()
if not actor:
return {'success': False, 'message': "Could not find actor"}
actor.name = name.strip()
actor.save(username=user)
return {'success': True}
def set_actor_description(id_, description, analyst):
"""
Set an Actor description.
:param id_: Actor ObjectId.
:type id_: str
:param description: The new description.
:type description: str
:param analyst: The user updating the description.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
sources = user_sources(analyst)
actor = Actor.objects(id=id_, source__name__in=sources).first()
if not actor:
return {'success': False, 'message': "Could not find actor"}
actor.description = description.strip()
actor.save(username=analyst)
return {'success': True}
def update_title(type_, id_, title, user, **kwargs):
"""
Change signature data for the current version
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: The ObjectId to search for.
:type id_: str
:param title: The new signature title to use.
:type title: str
:param user: The user setting the data value.
:type user: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
# Have to unescape the submitted data. Use unescape() to escape
# < and friends. Use urllib2.unquote() to escape %3C and friends.
h = HTMLParser.HTMLParser()
data = h.unescape(title)
try:
obj.title = data
obj.save(username=title)
return {'success': True, 'message': "Signature title updated."}
except ValidationError, e:
return {'success': False, 'message': e}
def update_max_version(type_, id_, data_type_max_version, user, **kwargs):
"""
Change the max version of the data tool
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: The ObjectId to search for.
:type id_: str
:param data_type_max_version: The new max version to use.
:type data_type_max_version: str
:param user: The user setting the description.
:type user: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
# Have to unescape the submitted data. Use unescape() to escape
# < and friends. Use urllib2.unquote() to escape %3C and friends.
h = HTMLParser.HTMLParser()
data_type_max_version = h.unescape(data_type_max_version)
try:
obj.data_type_max_version = data_type_max_version
obj.save(username=user.username)
return {'success': True, 'message': "Data type max version set."}
except ValidationError, e:
return {'success': False, 'message': e}
def indicator_from_header_field(request, email_id):
"""
Create an indicator from a header field. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to get the header from.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
if 'type' in request.POST:
header_field = request.POST['type']
analyst = request.user.username
sources = user_sources(analyst)
email = Email.objects(id=email_id,
source__name__in=sources).first()
if not email:
result = {'success': False, 'message': "Could not find email."}
else:
if header_field in ("from_address, sender, reply_to"):
ind_type = "Address - e-mail"
elif header_field in ("originating_ip", "x_originating_ip"):
ind_type = "Address - ipv4-addr"
else:
ind_type = "String"
result = create_indicator_from_header_field(
email, header_field, ind_type, analyst, request)
else:
result = {'success': False, 'message': "Type is a required value."}
return HttpResponse(json.dumps(result), mimetype="application/json")
else:
return render_to_response('error.html',
{'error': "Expected AJAX POST"},
RequestContext(request))
def generate_timeline(obj_type, obj_id, user):
users_sources = user_sources(user)
obj_class = class_from_type(obj_type)
if hasattr(obj_class, 'source'):
main_obj = obj_class.objects(id=obj_id,
source__name__in=users_sources).first()
else:
main_obj = obj_class.objects(id=obj_id).first()
if not main_obj:
return {'success': False, 'message': 'No starting object found.'}
# timeline is a dictionary.
# the key is the date with no time allowing us to collect a day's events.
# the value is a list of tuples.
# the first item in the tuple should be a datetime string for the event.
# the second element should be a description of the event that happened.
timeline = {}
# creation time
i = "<b>%s</b> was created" % obj_type
append_to_timeline(timeline, main_obj.created, i)
# sources
if hasattr(main_obj, 'source'):
for source in main_obj.source:
if source.name in users_sources:
name = source.name
for instance in source.instances:
i = "Source <b>%s</b> provided %s with a method of <b>'%s'</b> \
and a reference of <b>'%s'</b>" % (
name, obj_type, instance.method, instance.reference)
append_to_timeline(timeline, instance.date, i)
# releasability
for release in main_obj.releasability:
if release.name in users_sources:
name = release.name
for instance in release.instances:
i = "Release to <b>%s</b> added." % name
append_to_timeline(timeline, instance.date, i)
# campaigns
for campaign in main_obj.campaign:
name = campaign.name
confidence = campaign.confidence
description = campaign.description
rev = reverse('crits.campaigns.views.campaign_details', args=[
name,
])
link = '<a href="%s">%s</a>' % (rev, name)
i = "Campaign <b>%s</b> added with a confidence of <b>%s</b> and a \
description of '%s'" % (link, confidence, description)
append_to_timeline(timeline, campaign.date, i)
# objects
for obj in main_obj.obj:
name = obj.name
type_ = obj.object_type
if name == type_:
object_type = name
else:
object_type = "%s - %s" % (name, type_)
value = obj.value
rev = '%s?search_type=object&otype=%s&q=%s&force_full=1' \
% (reverse('crits.core.views.global_search_listing'),
"%s - %s" % (type_, name),
urllib.quote(value))
link = '<a href="%s">%s</a>' % (rev, value)
i = "<b>%s</b> object added with a value of :<br />%s" % (object_type,
link)
append_to_timeline(timeline, obj.date, i)
# relationships
for rel in main_obj.relationships:
tobj = class_from_type(rel.rel_type)
if tobj.objects(id=rel.object_id,
source__name__in=users_sources).only('id').first():
rev = reverse('crits.core.views.details',
args=[
rel.rel_type,
str(rel.object_id),
])
link = '<a href="%s">%s</a>' % (rev, rel.rel_type)
i = "<b>%s</b> was added with a relationship of <b>%s</b>." % (
link, rel.relationship)
append_to_timeline(timeline, rel.date, i)
# comments
cobj = class_from_type("Comment")
comments = cobj.objects(obj_type=obj_type,
obj_id=obj_id,
comment_type="comment")
for comment in comments:
i = "<b>%s</b> made a comment: %s" % (comment.analyst, comment.comment)
append_to_timeline(timeline, comment.created, i)
# analysis
for analysis in main_obj.analysis:
analyst = analysis.analyst
service_name = analysis.service_name
version = analysis.version
results = len(analysis.results)
i = "<b>%s</b> ran <b>%s (%s)</b> and got <b>%d</b> results." % (
analyst, service_name, version, results)
append_to_timeline(timeline, analysis.start_date, i)
# tickets
for ticket in main_obj.tickets:
i = "<b>%s</b> added Ticket <b>%s</b>" % (ticket.analyst,
ticket.ticket_number)
append_to_timeline(timeline, ticket.date, i)
# raw data specific timeline entries
if obj_type == "RawData":
# inline comments
for inline in main_obj.inlines:
i = "<b>%s</b> made an inline comment on line <b>%d</b>: %s" % (
inline.analyst, inline.line, inline.comment)
append_to_timeline(timeline, inline.date, i)
# highlights
for highlight in main_obj.highlights:
i = "<b>%s</b> highlighted line <b>%d</b>: %s" % (
highlight.analyst, highlight.line, highlight.comment)
append_to_timeline(timeline, highlight.date, i)
# versions
robj = class_from_type(obj_type)
versions = robj.objects(link_id=main_obj.link_id).only(
'id', 'version', 'created')
for version in versions:
rev = reverse('crits.raw_data.views.raw_data_details',
args=[
str(version.id),
])
link = '<a href="%s">%d</a>' % (rev, version.version)
i = "Version %s was added." % link
append_to_timeline(timeline, version.created, i)
# indicator specific timeline entries
if obj_type == "Indicator":
# actions
for action in main_obj.actions:
i = "<b>%s</b> added action <b>%s</b> to start on <b>%s</b>" \
% (action.analyst,
action.action_type,
action.begin_date)
i += ", set to <b>%s</b>, with a reason of: <b>%s</b>" \
% (action.active,
action.reason)
append_to_timeline(timeline, action.date, i)
# activity
for activity in main_obj.activity:
i = "<b>%s</b> noted Indicator activity from <b>%s</b> to <b>%s</b> \
and said: %s" % (activity.analyst, activity.start_date,
activity.end_date, activity.description)
append_to_timeline(timeline, activity.date, i)
# sort timeline
sorted_timeline = []
keys = timeline.keys()
keys.sort()
for key in keys:
k = timeline[key]
k.sort(key=lambda tup: tup[0])
sorted_timeline.append((key, k))
html = render_to_string('timeline_contents.html',
{'timeline': sorted_timeline})
return {'success': True, 'message': html}
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id,
'Campaign')
# subscription
subscription = {
'type':
'Campaign',
'id':
campaign_detail.id,
'subscribed':
is_user_subscribed("%s" % analyst, 'Campaign', campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {
'comments': campaign_detail.get_comments(),
'url_key': campaign_name
}
#screenshots
screenshots = campaign_detail.get_screenshots(analyst)
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(
source__name__in=sources, __raw__=formatted_query).count()
# Item counts for targets
uniq_addrs = get_campaign_targets(campaign_name, analyst)
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
# analysis results
service_results = campaign_detail.get_analysis_results()
args = {
'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"screenshots": screenshots,
'service_results': service_results,
"ttp_form": ttp_form
}
return template, args
def create_notification(obj, username, message, source_filter=None,
notification_type=NotificationType.ALERT):
"""
Generate a notification -- based on mongo obj.
:param obj: The object.
:type obj: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param username: The user creating the notification.
:type username: str
:param message: The notification message.
:type message: str
:param source_filter: Filter on who can see this notification.
:type source_filter: list(str)
:param notification_type: The notification type (e.g. alert, error).
:type notification_type: str
"""
n = Notification()
n.analyst = username
obj_type = obj._meta['crits_type']
if notification_type not in NotificationType.ALL:
notification_type = NotificationType.ALERT
n.notification_type = notification_type
if obj_type == 'Comment':
n.obj_id = obj.obj_id
n.obj_type = obj.obj_type
n.notification = "%s added a comment: %s" % (username, obj.comment)
else:
n.notification = message
n.obj_id = obj.id
n.obj_type = obj_type
if hasattr(obj, 'source'):
sources = [s.name for s in obj.source]
if obj_type == 'Comment':
# for comments, use the sources from the object that it is linked to
# instead of the comments's sources
referenced_obj = class_from_id(n.obj_type, n.obj_id)
sources = [s.name for s in referenced_obj.source]
subscribed_users = get_subscribed_users(n.obj_type, n.obj_id, sources)
# Filter on users that have access to the source of the object
for subscribed_user in subscribed_users:
allowed_sources = user_sources(subscribed_user)
for allowed_source in allowed_sources:
if allowed_source in sources:
if source_filter is None or allowed_source in source_filter:
n.users.append(subscribed_user)
break
else:
n.users = get_subscribed_users(n.obj_type, n.obj_id, [])
if obj_type == 'Comment':
for u in obj.users:
if u not in n.users:
n.users.append(u)
# don't notify the user creating this notification
n.users = [u for u in n.users if u != username]
if not len(n.users):
return
try:
n.save()
except ValidationError:
pass
# Signal potentially waiting threads that notification information is available
for user in n.users:
notification_lock = NotificationLockManager.get_notification_lock(user)
notification_lock.acquire()
try:
notification_lock.notifyAll()
finally:
notification_lock.release()
def get_event_details(event_id, user):
"""
Generate the data to render the Event details template.
:param event_id: The ObjectId of the Event to get details for.
:type event_id: str
:param user: The user requesting this information.
:type user: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(user)
event = Event.objects(id=event_id, source__name__in=sources).first()
if not user.check_source_tlp(event):
event = None
if not event:
template = "error.html"
args = {'error': "ID does not exist or insufficient privs for source"}
return template, args
event.sanitize("%s" % user)
campaign_form = CampaignForm()
download_form = DownloadFileForm(initial={
"obj_type": 'Event',
"obj_id": event_id
})
# remove pending notifications for user
remove_user_from_notification("%s" % user, event.id, 'Event')
# subscription
subscription = {
'type': 'Event',
'id': event.id,
'subscribed': is_user_subscribed("%s" % user, 'Event', event.id),
}
#objects
objects = event.sort_objects()
#relationships
relationships = event.sort_relationships("%s" % user, meta=True)
# Get count of related Events for each related Indicator
for ind in relationships.get('Indicator', []):
count = Event.objects(relationships__object_id=ind['id'],
source__name__in=sources).count()
ind['rel_ind_events'] = count
# Get count of related Events for each related Sample
for smp in relationships.get('Sample', []):
count = Event.objects(relationships__object_id=smp['id'],
source__name__in=sources).count()
smp['rel_smp_events'] = count
# relationship
relationship = {'type': 'Event', 'value': event.id}
#comments
comments = {'comments': event.get_comments(), 'url_key': event.id}
#screenshots
screenshots = event.get_screenshots(user)
# favorites
favorite = is_user_favorite("%s" % user, 'Event', event.id)
# services
service_list = get_supported_services('Event')
# analysis results
service_results = event.get_analysis_results()
args = {
'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'screenshots': screenshots,
'event': event,
'campaign_form': campaign_form,
'service_results': service_results,
'download_form': download_form,
'EventACL': EventACL
}
return template, args
def get_certificate_details(md5, user):
"""
Generate the data to render the Certificate details template.
:param md5: The MD5 of the Certificate to get details for.
:type md5: str
:param user: The user requesting this information.
:type user: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(user.username)
cert = Certificate.objects(md5=md5, source__name__in=sources).first()
if not user.check_source_tlp(cert):
cert = None
if not cert:
template = "error.html"
args = {
'error':
'Certificate not yet available or you do not have access to view it.'
}
else:
cert.sanitize("%s" % user.username)
# remove pending notifications for user
remove_user_from_notification("%s" % user.username, cert.id,
'Certificate')
# subscription
subscription = {
'type':
'Certificate',
'id':
cert.id,
'subscribed':
is_user_subscribed("%s" % user.username, 'Certificate', cert.id),
}
#objects
objects = cert.sort_objects()
#relationships
relationships = cert.sort_relationships("%s" % user.username,
meta=True)
# relationship
relationship = {'type': 'Certificate', 'value': cert.id}
#comments
comments = {'comments': cert.get_comments(), 'url_key': md5}
#screenshots
screenshots = cert.get_screenshots(user.username)
# services
service_list = get_supported_services('Certificate')
# analysis results
service_results = cert.get_analysis_results()
args = {
'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'relationship': relationship,
"subscription": subscription,
"screenshots": screenshots,
'service_results': service_results,
"cert": cert,
"CertificateACL": CertificateACL,
}
return template, args
def add_sample_for_event(event_id,
data,
analyst,
filedata=None,
filename=None,
md5=None,
email_addr=None,
inherit_sources=False):
"""
Add a sample related to this Event.
:param event_id: The ObjectId of the Event to associate with.
:type event_id: str
:param data: The form data.
:type data: dict
:param analyst: The user adding this Sample.
:type analyst: str
:param filedata: The sample data.
:type filedata: file handle.
:param filename: The name of the file.
:type filename: str
:param md5: The MD5 of the file.
:type md5: str
:param email_addr: Email address to which to email the sample
:type email_addr: str
:param inherit_sources: 'True' if Sample should inherit Event's Source(s)
:type inherit_sources: bool
:returns: dict with keys "success" (boolean) and "message" (str)
"""
response = {
'success': False,
'message': 'Unknown error; unable to upload file.'
}
users_sources = user_sources(analyst)
event = Event.objects(id=event_id, source__name__in=users_sources).first()
if not event:
return {'success': False, 'message': "No matching event found"}
source = data['source']
reference = data['reference']
file_format = data['file_format']
campaign = data['campaign']
confidence = data['confidence']
bucket_list = data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
ticket = data[form_consts.Common.TICKET_VARIABLE_NAME]
method = data['method']
if filename:
filename = filename.strip()
# If selected, new sample inherits the campaigns of the related event.
if data['inherit_campaigns']:
if campaign:
event.campaign.append(
EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=analyst))
campaign = event.campaign
inherited_source = event.source if inherit_sources else None
try:
if filedata:
result = handle_uploaded_file(filedata,
source,
method,
reference,
file_format,
data['password'],
analyst,
campaign,
confidence,
related_id=event.id,
related_type='Event',
filename=filename,
bucket_list=bucket_list,
ticket=ticket,
inherited_source=inherited_source)
else:
if md5:
md5 = md5.strip().lower()
result = handle_uploaded_file(None,
source,
method,
reference,
file_format,
None,
analyst,
campaign,
confidence,
related_id=event.id,
related_type='Event',
filename=filename,
md5=md5,
bucket_list=bucket_list,
ticket=ticket,
inherited_source=inherited_source,
is_return_only_md5=False)
except ZipFileError, zfe:
return {'success': False, 'message': zfe.value}
def get_ip_details(ip, analyst):
"""
Generate the data to render the IP details template.
:param ip: The IP to get details for.
:type ip: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
allowed_sources = user_sources(analyst)
ip = IP.objects(ip=ip, source__name__in=allowed_sources).first()
template = None
args = {}
if not ip:
template = "error.html"
error = ('Either no data exists for this IP or you do not have'
' permission to view it.')
args = {'error': error}
else:
ip.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, ip.id, 'IP')
# subscription
subscription = {
'type': 'IP',
'id': ip.id,
'subscribed': is_user_subscribed("%s" % analyst, 'IP', ip.id),
}
#objects
objects = ip.sort_objects()
#relationships
relationships = ip.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {
'type': 'IP',
'value': ip.id
}
#comments
comments = {'comments': ip.get_comments(),
'url_key':ip.ip}
#screenshots
screenshots = ip.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'IP', ip.id)
# services
manager = crits.service_env.manager
service_list = manager.get_supported_services('IP', True)
args = {'objects': objects,
'relationships': relationships,
'relationship': relationship,
'subscription': subscription,
'favorite': favorite,
'service_list': service_list,
'screenshots': screenshots,
'ip': ip,
'comments':comments}
return template, args
def get_actor_details(id_, analyst):
"""
Generate the data to render the Actor details template.
:param id_: The Actor ObjectId to get details for.
:type actorip: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
allowed_sources = user_sources(analyst)
actor = Actor.objects(id=id_, source__name__in=allowed_sources).first()
template = None
args = {}
if not actor:
template = "error.html"
error = ('Either no data exists for this Actor or you do not have'
' permission to view it.')
args = {'error': error}
else:
actor.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, actor.id, 'Actor')
download_form = DownloadFileForm(initial={
"obj_type": 'Actor',
"obj_id": actor.id
})
# generate identifiers
actor_identifiers = actor.generate_identifiers_list(analyst)
# subscription
subscription = {
'type': 'Actor',
'id': actor.id,
'subscribed': is_user_subscribed("%s" % analyst, 'Actor',
actor.id),
}
#objects
objects = actor.sort_objects()
#relationships
relationships = actor.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'Actor', 'value': actor.id}
#comments
comments = {'comments': actor.get_comments(), 'url_key': actor.id}
#screenshots
screenshots = actor.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Actor', actor.id)
# services
service_list = get_supported_services('Actor')
# analysis results
service_results = actor.get_analysis_results()
args = {
'actor_identifiers': actor_identifiers,
'objects': objects,
'download_form': download_form,
'relationships': relationships,
'relationship': relationship,
'subscription': subscription,
'favorite': favorite,
'service_list': service_list,
'service_results': service_results,
'screenshots': screenshots,
'actor': actor,
'actor_id': id_,
'comments': comments
}
return template, args
def get_object_list(self, request, klass, sources=True):
"""
Handle GET requests. This does all sorts of work to ensure the
results are sanitized and that source restriction is adhered to.
Adds the ability to limit results and the content of the results
through GET parameters.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param klass: The CRITs top-level object to get.
:type klass: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param sources: If we should limit by source.
:type sources: boolean
:returns: :class:`crits.core.crits_mongoengine.CritsQuerySet`
"""
querydict = {}
get_params = request.GET.copy()
regex = request.GET.get('regex', False)
only = request.GET.get('only', None)
exclude = request.GET.get('exclude', None)
source_list = user_sources(request.user.username)
no_sources = True
for k,v in get_params.iteritems():
v = v.strip()
try:
v_int = int(v)
except:
pass
if k == "c-_id":
try:
querydict['_id'] = ObjectId(v)
except:
pass
if k.startswith("c-"):
field = k[2:]
# Attempt to discover query operators. We use django-style operators
# (same as MongoEngine). These also override regex.
try:
op_index = field.index("__")
op = "$%s" % field[op_index+2:]
field = field[:op_index]
except ValueError:
op_index = None
if op_index is not None:
if op in ('$gt', '$gte', '$lt', '$lte', '$ne', '$in', '$nin'):
val = v
if op in ('$in', '$nin'):
if field == 'source.name':
val = []
for i in v.split(','):
s = remove_quotes(i)
if s in source_list:
no_sources = False
val.append(s)
else:
val = [remove_quotes(i) for i in v.split(',')]
if field in ('size', 'schema_version'):
if isinstance(val, list):
v_f = []
for i in val:
try:
v_f.append(int(i))
except:
pass
val = v_f
else:
try:
val = int(val)
except:
val = None
if val:
querydict[field] = {op: val}
elif field in ('size', 'schema_version'):
querydict[field] = v_int
elif field == 'source.name':
v = remove_quotes(v)
if v in source_list:
no_sources = False
querydict[field] = v
elif regex:
querydict[field] = generate_regex(v)
else:
querydict[field] = remove_quotes(v)
if no_sources and sources:
querydict['source.name'] = {'$in': source_list}
if only or exclude:
required = [k for k,v in klass._fields.iteritems() if v.required]
if only:
fields = only.split(',')
if exclude:
excludes = exclude.split(',')
fields = [x for x in fields if x not in excludes]
for r in required:
if r not in fields:
fields.append(r)
results = klass.objects(__raw__=querydict).only(*fields)
elif exclude:
fields = exclude.split(',')
for r in required:
if r not in fields:
fields.append(r)
results = klass.objects(__raw__=querydict).exclude(*fields)
else:
results = klass.objects(__raw__=querydict)
return results
def get_object_list(self, request, klass, sources=True):
"""
Handle GET requests. This does all sorts of work to ensure the
results are sanitized and that source restriction is adhered to.
Adds the ability to limit results and the content of the results
through GET parameters.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param klass: The CRITs top-level object to get.
:type klass: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param sources: If we should limit by source.
:type sources: boolean
:returns: :class:`crits.core.crits_mongoengine.CritsQuerySet`
"""
querydict = {}
get_params = request.GET.copy()
regex = request.GET.get('regex', False)
only = request.GET.get('only', None)
exclude = request.GET.get('exclude', None)
source_list = user_sources(request.user.username)
no_sources = True
# Chop off trailing slash and split on remaining slashes.
# If last part of path is not the resource name, assume it is an
# object ID.
path = request.path[:-1].split('/')
if path[-1] != self.Meta.resource_name:
# If this is a valid object ID, convert it. Otherwise, use
# the string. The corresponding query will return 0.
if ObjectId.is_valid(path[-1]):
querydict['_id'] = ObjectId(path[-1])
else:
querydict['_id'] = path[-1]
do_or = False
for k, v in get_params.iteritems():
v = v.strip()
try:
v_int = int(v)
except:
# If can't be converted to an int use the string.
v_int = v
if k == "c-_id":
try:
querydict['_id'] = ObjectId(v)
except:
pass
if k.startswith("c-"):
field = k[2:]
# Attempt to discover query operators. We use django-style operators
# (same as MongoEngine). These also override regex.
try:
op_index = field.index("__")
op = "$%s" % field[op_index + 2:]
field = field[:op_index]
except ValueError:
op_index = None
if op_index is not None:
if op in ('$gt', '$gte', '$lt', '$lte', '$ne', '$in',
'$nin', '$exists'):
val = v
if field in ('created', 'modified'):
try:
val = parse(val, fuzzy=True)
except:
pass
if op in ('$in', '$nin'):
if field == 'source.name':
val = []
for i in v.split(','):
s = remove_quotes(i)
if s in source_list:
no_sources = False
val.append(s)
else:
val = [remove_quotes(i) for i in v.split(',')]
if op == '$exists':
if val in ('true', 'True', '1'):
val = 1
elif val in ('false', 'False', '0'):
val = 0
if field in ('size', 'schema_version'):
if isinstance(val, list):
v_f = []
for i in val:
try:
v_f.append(int(i))
except:
pass
val = v_f
else:
try:
val = int(val)
except:
val = None
if val or val == 0:
querydict[field] = {op: val}
elif field in ('size', 'schema_version'):
querydict[field] = v_int
elif field in ('created', 'modified'):
try:
querydict[field] = parse(v, fuzzy=True)
except:
querydict[field] = v
elif field == 'source.name':
v = remove_quotes(v)
if v in source_list:
no_sources = False
querydict[field] = v
elif regex:
querydict[field] = generate_regex(v)
else:
querydict[field] = remove_quotes(v)
if k == 'or':
do_or = True
if do_or:
tmp = {}
tmp['$or'] = [{x: y} for x, y in querydict.iteritems()]
querydict = tmp
if no_sources and sources:
querydict['source.name'] = {'$in': source_list}
if only or exclude:
required = [k for k, f in klass._fields.iteritems() if f.required]
if only:
fields = only.split(',')
if exclude:
excludes = exclude.split(',')
fields = [x for x in fields if x not in excludes]
for r in required:
if r not in fields:
fields.append(r)
results = klass.objects(__raw__=querydict).only(*fields)
elif exclude:
fields = exclude.split(',')
for r in required:
if r not in fields:
fields.append(r)
results = klass.objects(__raw__=querydict).exclude(*fields)
else:
results = klass.objects(__raw__=querydict)
return results
def gather_relationships(obj_type, obj_id, user, depth, types):
objects = {}
nodes = []
links = []
# These would be used if we move to force labels
#labelAnchors = []
#labelAnchorLinks = []
sources = user_sources(user)
if not sources:
return { 'nodes': nodes, 'links': links }
field_dict = {
'Actor': 'name',
'Campaign': 'name',
'Certificate': 'md5',
'Comment': 'object_id',
'Domain': 'domain',
'Email': 'date',
'Event': 'title',
'Indicator': 'value',
'IP': 'ip',
'PCAP': 'md5',
'RawData': 'title',
'Sample': 'md5',
'Target': 'email_address'
}
# color scheme:
# http://colorschemedesigner.com/#00426p4O9CCPc
color_dict = {
'Actor': '#900C0C',
'Campaign': '#FF3737',
'Certificate': '#FFA837',
'Comment': '#3A98DA',
'Domain': '#33EB33',
'Email': '#FF8989',
'Event': '#B05151',
'Indicator': '#B08751',
'IP': '#90570C',
'PCAP': '#FFCC89',
'RawData': '#4A7797',
'Sample': '#8CCBF8',
'Target': '#4AA24A'
}
def inner_collect(obj_type, obj_id, sources, depth):
# Don't keep going if we've already processed this object
if obj_id in objects:
return
klass = class_from_type(obj_type)
if not klass:
return
if hasattr(klass, 'source'):
obj = klass.objects(id=obj_id, source__name__in=sources).first()
else:
obj = klass.objects(id=obj_id).first()
if not obj:
return
objects[obj_id] = obj
if depth == 0:
return
depth -= 1
for r in obj.relationships:
inner_collect(r.rel_type, str(r.object_id), sources, depth)
# If we traverse into a Campaign object, walk everything tagged
# with that campaign along with related objects.
if obj_type == 'Campaign':
for c in field_dict.keys():
klass = class_from_type(c)
# Not every object in field_dict can be tagged with a campaign.
# For example, comments.
if not hasattr(klass, 'campaign'):
continue
tagged_objs = klass.objects(campaign__name=obj.name)
for tobj in tagged_objs:
inner_collect(tobj._meta['crits_type'],
str(tobj.id),
sources,
depth)
# END OF INNER COLLECT
try:
depth = int(depth)
except ValueError:
depth = 3
inner_collect(obj_type, str(obj_id), sources, depth)
# This dictionary is used to describe the position of each object
# in the nodes list. The key is an object ID and the value is a
# tuple where the first item is the position in the node list and
# the second is a list of object IDs related to this object.
#
# Once all the nodes are created and this dictionary is populated
# it is used to create the links.
#
# Here is a simple obj_graph for three objects (A, B and C) that
# are related such that they form a triangle (A<->B, B<->C, C<->A):
#
# {
# 'A': (0, ['B', 'C']),
# 'B': (1, ['A', 'C']),
# 'C': (2, ['B', 'A'])
# }
#
# The integers are the position in the node array and are used as
# the target. The items in the list are used as lookups back into
# the dictionary for the source.
obj_graph = {}
node_position = 0
for (obj_id, obj) in objects.iteritems():
if obj_id in obj_graph:
continue
obj_type = obj._meta['crits_type']
value = getattr(obj, field_dict[obj_type], '')
href = reverse('crits.core.views.details', args=(obj_type, obj_id))
if len(types) != 0 and obj_type not in types:
color = "#FFFFFF"
visible = False
else:
color = color_dict[obj_type]
visible = True
# For every campaign on this object, make a new node in the list.
if hasattr(obj, 'campaign'):
for i, campaign in enumerate(obj.campaign):
name = "%s" % obj.campaign[i].name
(x, campaign_details) = get_campaign_details(name, user)
if 'error' in campaign_details:
continue
campaign_id = str(campaign_details['campaign_detail'].id)
# If this campaign already exists as a node then
# add a relationship to the current object
if campaign_id in obj_graph:
(tnode, source_ids) = obj_graph[campaign_id]
source_ids.append(obj_id)
else:
total = 0
for count in campaign_details['counts'].values():
total += count
campaign = name + " (" + str(total) + ")"
campaign_href = reverse('crits.core.views.details', args=('Campaign', campaign_id))
campaign_color = color_dict['Campaign']
n = {
'label': campaign,
'url': campaign_href,
'color': campaign_color,
'type': 'Campaign',
'visible': True
}
nodes.append(n)
obj_graph[campaign_id] = (node_position, [obj_id])
node_position += 1
n = {
'label': '%s' % value,
'url': href,
'color': color,
'type': obj_type,
'visible': visible
}
nodes.append(n)
obj_graph[obj_id] = (node_position, [str(r.object_id) for r in obj.relationships])
node_position += 1
# This dictionary is used to track the links that have been created.
# When a new link is created the inverse is added to this dictionary as
# a key. This is because the link between A->B is the same as B->A. When
# the link for A->B is made, we insert it into this dictionary and then
# lookup the inverse when creating any new relationships. This ensures
# that when B->A is handled it will be ignored.
link_dict = {}
for (tid, (tnode, source_ids)) in obj_graph.iteritems():
for sid in source_ids:
# If the graph is cut off the related object may not have been
# collected. If the inverse relationship is already done, no
# need to do this one too.
if sid not in obj_graph or (tid + sid) in link_dict:
continue
link = {
'source': obj_graph[sid][0],
'target': tnode,
'weight': 1,
}
links.append(link)
link_dict[sid + tid] = True
#labelAnchors.append({'node': n,
# 'url': href})
#labelAnchors.append({'node': n,
# 'url': href})
#alink = {
# 'source': (len(nodes) - 1) * 2,
# 'target': (len(nodes) - 1) * 2 + 1,
# 'weight': 1,
#}
#labelAnchorLinks.append(alink)
return {
'nodes': nodes,
'links': links,
#'labelAnchors': labelAnchors,
#'labelAnchorLinks': labelAnchorLinks,
}
def get_domain_details(domain, analyst):
"""
Generate the data to render the Domain details template.
:param domain: The name of the Domain to get details for.
:type domain: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
dmain = Domain.objects(domain=domain,
source__name__in=allowed_sources).first()
if not dmain:
error = ("Either no data exists for this domain"
" or you do not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
forms = {}
#populate whois data into whois form
# and create data object (keyed on date) for updating form on date select
whois_data = {'': ''} #blank info for "Add New" option
initial_data = {'data': ' '}
raw_data = {}
whois = getattr(dmain, 'whois', None)
if whois:
for w in whois:
#build data as a display-friendly string
w.date = datetime.datetime.strftime(w.date,
settings.PY_DATETIME_FORMAT)
from whois_parser import WhoisEntry
#prettify the whois data
w.data = unicode(WhoisEntry.from_dict(w.data))
if 'text' not in w: #whois data was added with old data format
w.text = w.data
#also save our text blob for easy viewing of the original data
whois_data[w.date] = (w.data, w.text)
#show most recent entry first
initial_data = {'data': whois[-1].data, 'date': whois[-1].date}
raw_data = {'data': whois[-1].text, 'date': whois[-1].date}
whois_len = len(
whois_data) - 1 #subtract one to account for blank "Add New" entry
whois_data = json.dumps(whois_data)
dmain.sanitize_sources(username="******" % analyst, sources=allowed_sources)
forms['whois'] = UpdateWhoisForm(initial_data, domain=domain)
forms['raw_whois'] = UpdateWhoisForm(raw_data,
domain=domain,
allow_adding=False)
forms['diff_whois'] = DiffWhoisForm(domain=domain)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, dmain.id, 'Domain')
# subscription
subscription = {
'type': 'Domain',
'id': dmain.id,
'subscribed': is_user_subscribed("%s" % analyst, 'Domain', dmain.id),
}
#objects
objects = dmain.sort_objects()
#relationships
relationships = dmain.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'Domain', 'value': dmain.id}
#comments
comments = {'comments': dmain.get_comments(), 'url_key': dmain.domain}
#screenshots
screenshots = dmain.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Domain', dmain.id)
# services
manager = crits.service_env.manager
service_list = manager.get_supported_services('Domain', True)
args = {
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'screenshots': screenshots,
'domain': dmain,
'forms': forms,
'whois_data': whois_data,
'service_list': service_list,
'whois_len': whois_len
}
return template, args
def upload_attach(request, email_id):
"""
Upload an attachment for an email.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:param email_id: The ObjectId of the email to upload attachment for.
:type email_id: str
:returns: :class:`django.http.HttpResponse`
"""
redirect = reverse('crits.emails.views.email_detail', args=[email_id])
user = request.user
if request.method != 'POST':
return HttpResponseRedirect(redirect)
file_form = UploadFileForm(request.user, request.POST, request.FILES)
json_reply = {'success': False}
if not file_form.is_valid():
file_form.fields['related_md5_event'].widget = forms.HiddenInput() #hide field so it doesn't reappear
json_reply['form'] = file_form.as_table()
return render_to_response('file_upload_response.html',
{'response': json.dumps(json_reply)},
RequestContext(request))
if not user.has_access_to(EmailACL.ADD_ATTACHMENT):
json_reply['message'] = "User does not have permission to upload attachment."
return render_to_response('file_upload_response.html',
{'response': json.dumps(json_reply)},
RequestContext(request))
analyst = request.user.username
users_sources = user_sources(analyst)
method = file_form.cleaned_data['method'] or "Add to Email"
bucket_list = file_form.cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = file_form.cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
email_addr = None
if request.POST.get('email'):
email_addr = request.user.email
email = Email.objects(id=email_id,
source__name__in=users_sources).first()
if not email:
json_reply['message'] = "Could not find email."
return render_to_response('file_upload_response.html',
{'response': json.dumps(json_reply)},
RequestContext(request))
result = create_email_attachment(email,
file_form,
analyst,
file_form.cleaned_data['source'],
method,
file_form.cleaned_data['reference'],
file_form.cleaned_data['campaign'],
file_form.cleaned_data['confidence'],
bucket_list,
ticket,
request.FILES.get('filedata'),
request.POST.get('filename'),
request.POST.get('md5'),
email_addr,
file_form.cleaned_data['inherit_sources'])
# If successful, tell the browser to redirect back to this email.
if result['success']:
result['redirect_url'] = redirect
return render_to_response('file_upload_response.html',
{'response': json.dumps(result)},
RequestContext(request))
def get_event_details(event_id, analyst):
"""
Generate the data to render the Event details template.
:param event_id: The ObjectId of the Event to get details for.
:type event_id: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
event = Event.objects(id=event_id, source__name__in=sources).first()
if not event:
template = "error.html"
args = {'error': "ID does not exist or insufficient privs for source"}
return template, args
event.sanitize("%s" % analyst)
campaign_form = CampaignForm()
download_form = DownloadFileForm(initial={
"obj_type": 'Event',
"obj_id": event_id
})
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, event.id, 'Event')
# subscription
subscription = {
'type': 'Event',
'id': event.id,
'subscribed': is_user_subscribed("%s" % analyst, 'Event', event.id),
}
#objects
objects = event.sort_objects()
#relationships
relationships = event.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'Event', 'value': event.id}
#comments
comments = {'comments': event.get_comments(), 'url_key': event.id}
#screenshots
screenshots = event.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Event', event.id)
# services
service_list = get_supported_services('Event')
# analysis results
service_results = event.get_analysis_results()
args = {
'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'screenshots': screenshots,
'event': event,
'campaign_form': campaign_form,
'service_results': service_results,
'download_form': download_form
}
return template, args
def get_indicator_details(indicator_id, analyst):
"""
Generate the data to render the Indicator details template.
:param indicator_id: The ObjectId of the Indicator to get details for.
:type indicator_id: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
users_sources = user_sources(analyst)
indicator = Indicator.objects(id=indicator_id,
source__name__in=users_sources).first()
if not indicator:
error = ("Either this indicator does not exist or you do "
"not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
forms = {}
forms['new_action'] = IndicatorActionsForm(initial={
'analyst': analyst,
'active': "off",
'date': datetime.datetime.now()
})
forms['new_activity'] = IndicatorActivityForm(
initial={
'analyst': analyst,
'date': datetime.datetime.now()
})
forms['new_campaign'] = CampaignForm() #'date': datetime.datetime.now(),
forms['new_source'] = SourceForm(analyst,
initial={'date': datetime.datetime.now()})
forms['download_form'] = DownloadFileForm(initial={
"obj_type": 'Indicator',
"obj_id": indicator_id
})
indicator.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, indicator_id, 'Indicator')
# subscription
subscription = {
'type':
'Indicator',
'id':
indicator_id,
'subscribed':
is_user_subscribed("%s" % analyst, 'Indicator', indicator_id),
}
# relationship
relationship = {
'type': 'Indicator',
'value': indicator_id,
}
#objects
objects = indicator.sort_objects()
#relationships
relationships = indicator.sort_relationships("%s" % analyst, meta=True)
#comments
comments = {'comments': indicator.get_comments(), 'url_key': indicator_id}
# favorites
favorite = is_user_favorite("%s" % analyst, 'Indicator', indicator.id)
# services
manager = crits.service_env.manager
service_list = manager.get_supported_services('Indicator', True)
args = {
'objects': objects,
'relationships': relationships,
'comments': comments,
'relationship': relationship,
'subscription': subscription,
"indicator": indicator,
"forms": forms,
"indicator_id": indicator_id,
'service_list': service_list,
'favorite': favorite,
'rt_url': settings.RT_URL
}
return template, args
def get_domain_details(domain, analyst):
"""
Generate the data to render the Domain details template.
:param domain: The name of the Domain to get details for.
:type domain: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
dmain = Domain.objects(domain=domain,
source__name__in=allowed_sources).first()
if not dmain:
error = ("Either no data exists for this domain"
" or you do not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
dmain.sanitize_sources(username="******" % analyst, sources=allowed_sources)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, dmain.id, 'Domain')
# subscription
subscription = {
'type': 'Domain',
'id': dmain.id,
'subscribed': is_user_subscribed("%s" % analyst, 'Domain', dmain.id),
}
#objects
objects = dmain.sort_objects()
#relationships
relationships = dmain.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'Domain', 'value': dmain.id}
#comments
comments = {'comments': dmain.get_comments(), 'url_key': dmain.domain}
#screenshots
screenshots = dmain.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Domain', dmain.id)
# services
service_list = get_supported_services('Domain')
# analysis results
service_results = dmain.get_analysis_results()
args = {
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'screenshots': screenshots,
'domain': dmain,
'service_list': service_list,
'service_results': service_results
}
return template, args
def find_emails(self, username):
sources = user_sources(username)
emails = Email.objects(to=self.email_address, source__name__in=sources)
return emails
def get_pcap_details(md5, analyst):
"""
Generate the data to render the PCAP details template.
:param md5: The MD5 of the PCAP to get details for.
:type md5: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
pcap = PCAP.objects(md5=md5, source__name__in=sources).first()
if not pcap:
template = "error.html"
args = {
'error':
'PCAP not yet available or you do not have access to view it.'
}
else:
pcap.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, pcap.id, 'PCAP')
# subscription
subscription = {
'type': 'PCAP',
'id': pcap.id,
'subscribed': is_user_subscribed("%s" % analyst, 'PCAP', pcap.id),
}
#objects
objects = pcap.sort_objects()
#relationships
relationships = pcap.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'PCAP', 'value': pcap.id}
#comments
comments = {'comments': pcap.get_comments(), 'url_key': md5}
#screenshots
screenshots = pcap.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'PCAP', pcap.id)
# services
# Assume all PCAPs have the data available
service_list = get_supported_services('PCAP')
# analysis results
service_results = pcap.get_analysis_results()
args = {
'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
"subscription": subscription,
"screenshots": screenshots,
"service_results": service_results,
"pcap": pcap
}
return template, args
def add_results(object_type, object_id, analysis_id, result, type_, subtype,
analyst):
"""
Add multiple results to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param result: The list of result to append.
:type result: list of str
:param type_: The list of result types.
:type type_: list of str
:param subtype: The list of result subtypes.
:type subtype: list of str
:param analyst: The user updating the results.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
res = {'success': False}
if not object_type or not object_id or not analysis_id:
res['message'] = "Must supply object id/type and analysis id."
return res
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
res['message'] = "Could not find object to add results to."
return res
if not (result and type_ and subtype):
res['message'] = "Need a result, type, and subtype to add a result."
return res
if not (len(result) == len(type_) == len(subtype)):
res['message'] = "result, type, and subtype need to be the same length."
return res
# Update analysis results
final_list = []
for key, r in enumerate(result):
final = {}
final['subtype'] = subtype[key]
final['result'] = r
tmp = ast.literal_eval(type_[key])
for k in tmp:
final[k] = tmp[k]
final_list.append(final)
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(
push_all__results=final_list)
res['success'] = True
return res
def get_raw_data_details(_id, user):
"""
Generate the data to render the RawData details template.
:param _id: The ObjectId of the RawData to get details for.
:type _id: str
:param user: The user requesting this information.
:type user: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(user)
if not _id:
raw_data = None
else:
raw_data = RawData.objects(id=_id, source__name__in=sources).first()
if not user.check_source_tlp(raw_data):
raw_data = None
if not raw_data:
template = "error.html"
args = {'error': 'raw_data not yet available or you do not have access to view it.'}
else:
raw_data.sanitize("%s" % user)
# remove pending notifications for user
remove_user_from_notification("%s" % user, raw_data.id, 'RawData')
# subscription
subscription = {
'type': 'RawData',
'id': raw_data.id,
'subscribed': is_user_subscribed("%s" % user,
'RawData', raw_data.id),
}
#objects
objects = raw_data.sort_objects()
#relationships
relationships = raw_data.sort_relationships("%s" % user, meta=True)
# relationship
relationship = {
'type': 'RawData',
'value': raw_data.id
}
versions = len(RawData.objects(link_id=raw_data.link_id).only('id'))
#comments
comments = {'comments': raw_data.get_comments(),
'url_key': _id}
#screenshots
screenshots = raw_data.get_screenshots(user)
# favorites
favorite = is_user_favorite("%s" % user, 'RawData', raw_data.id)
# services
service_list = get_supported_services('RawData')
# analysis results
service_results = raw_data.get_analysis_results()
args = {'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
"subscription": subscription,
"screenshots": screenshots,
"versions": versions,
"service_results": service_results,
"raw_data": raw_data,
"RawDataACL": RawDataACL}
return template, args
def get_backdoor_details(id_, user):
"""
Generate the data to render the Backdoor details template.
:param id_: The Backdoor ObjectId to get details for.
:type id_: str
:param user: The user requesting this information.
:type user: str
:returns: template (str), arguments (dict)
"""
allowed_sources = user_sources(user)
backdoor = Backdoor.objects(id=id_, source__name__in=allowed_sources).first()
template = None
args = {}
if not backdoor:
template = "error.html"
error = ('Either no data exists for this Backdoor or you do not have'
' permission to view it.')
args = {'error': error}
else:
backdoor.sanitize("%s" % user)
# remove pending notifications for user
remove_user_from_notification("%s" % user, backdoor.id, 'Backdoor')
# subscription
subscription = {
'type': 'Backdoor',
'id': backdoor.id,
'subscribed': is_user_subscribed("%s" % user,
'Backdoor',
backdoor.id),
}
#objects
objects = backdoor.sort_objects()
#relationships
relationships = backdoor.sort_relationships("%s" % user, meta=True)
# relationship
relationship = {
'type': 'Backdoor',
'value': backdoor.id
}
#comments
comments = {'comments': backdoor.get_comments(),
'url_key': backdoor.id}
#screenshots
screenshots = backdoor.get_screenshots(user)
# favorites
favorite = is_user_favorite("%s" % user, 'Backdoor', backdoor.id)
# services
service_list = get_supported_services('Backdoor')
# analysis results
service_results = backdoor.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
'relationship': relationship,
'subscription': subscription,
'favorite': favorite,
'service_list': service_list,
'service_results': service_results,
'screenshots': screenshots,
'backdoor': backdoor,
'backdoor_id': id_,
'comments': comments}
return template, args
