def get_aggregate_comments(atype, value, username, date=None):
"""
Generate a list of comments for the aggregate view.
:param atype: How to limit the comments ("bytag", "byuser", "bycomment").
:type atype: str
:param value: If limiting by atype, the value to limit by.
:type value: str
:param username: The user getting the comments.
:type username: str
:param date: The specific date to get comments for.
:type date: datetime.datetime
:returns: list of :class:`cripts.comments.comment.Comment`
"""
results = None
if date:
end_date = date+datetime.timedelta(days=1)
query = {'date':{'$gte':date, '$lte':end_date}}
else:
query = {}
if atype == 'bytag':
query['tags'] = value
elif atype == 'byuser':
query['$or'] = [{'users':value}, {'analyst':value}]
elif atype == 'bycomment':
query['comment'] = {'$regex':value}
results = Comment.objects(__raw__=query)
sources = user_sources(username)
return get_user_allowed_comments(results, sources)
def get_aggregate_comments(atype, value, username, date=None):
"""
Generate a list of comments for the aggregate view.
:param atype: How to limit the comments ("bytag", "byuser", "bycomment").
:type atype: str
:param value: If limiting by atype, the value to limit by.
:type value: str
:param username: The user getting the comments.
:type username: str
:param date: The specific date to get comments for.
:type date: datetime.datetime
:returns: list of :class:`cripts.comments.comment.Comment`
"""
results = None
if date:
end_date = date + datetime.timedelta(days=1)
query = {'date': {'$gte': date, '$lte': end_date}}
else:
query = {}
if atype == 'bytag':
query['tags'] = value
elif atype == 'byuser':
query['$or'] = [{'users': value}, {'analyst': value}]
elif atype == 'bycomment':
query['comment'] = {'$regex': value}
results = Comment.objects(__raw__=query)
sources = user_sources(username)
return get_user_allowed_comments(results, sources)
def add_results(object_type, object_id, analysis_id, result, type_, subtype,
analyst):
"""
Add multiple results to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param result: The list of result to append.
:type result: list of str
:param type_: The list of result types.
:type type_: list of str
:param subtype: The list of result subtypes.
:type subtype: list of str
:param analyst: The user updating the results.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
res = {'success': False}
if not object_type or not object_id or not analysis_id:
res['message'] = "Must supply object id/type and analysis id."
return res
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
res['message'] = "Could not find object to add results to."
return res
if not(result and type_ and subtype):
res['message'] = "Need a result, type, and subtype to add a result."
return res
if not(len(result) == len(type_) == len(subtype)):
res['message'] = "result, type, and subtype need to be the same length."
return res
# Update analysis results
final_list = []
for key, r in enumerate(result):
final = {}
final['subtype'] = subtype[key]
final['result'] = r
tmp = ast.literal_eval(type_[key])
for k in tmp:
final[k] = tmp[k]
final_list.append(final)
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(push_all__results=final_list)
res['success'] = True
return res
def refresh_services(request, cripts_type, identifier):
"""
Refresh the Analysis tab with the latest information.
"""
response = {}
# Verify user can see results.
sources = user_sources(request.user.username)
klass = class_from_type(cripts_type)
if not klass:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response),
content_type="application/json")
if hasattr(klass, 'source'):
obj = klass.objects(id=identifier, source__name__in=sources).first()
else:
obj = klass.objects(id=identifier).first()
if not obj:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response),
content_type="application/json")
# Get analysis results.
results = AnalysisResult.objects(object_type=cripts_type,
object_id=identifier)
relationship = {'type': cripts_type, 'value': identifier}
subscription = {'type': cripts_type, 'id': identifier}
service_list = get_supported_services(cripts_type)
response['success'] = True
response['html'] = render_to_string(
"services_analysis_listing.html", {
'relationship': relationship,
'subscription': subscription,
'service_results': results,
'cripts_type': cripts_type,
'identifier': identifier,
'service_list': service_list
}, RequestContext(request))
return HttpResponse(json.dumps(response), content_type="application/json")
def refresh_services(request, cripts_type, identifier):
"""
Refresh the Analysis tab with the latest information.
"""
response = {}
# Verify user can see results.
sources = user_sources(request.user.username)
klass = class_from_type(cripts_type)
if not klass:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), content_type="application/json")
if hasattr(klass, 'source'):
obj = klass.objects(id=identifier,source__name__in=sources).first()
else:
obj = klass.objects(id=identifier).first()
if not obj:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), content_type="application/json")
# Get analysis results.
results = AnalysisResult.objects(object_type=cripts_type,
object_id=identifier)
relationship = {'type': cripts_type,
'value': identifier}
subscription = {'type': cripts_type,
'id': identifier}
service_list = get_supported_services(cripts_type)
response['success'] = True
response['html'] = render_to_string("services_analysis_listing.html",
{'relationship': relationship,
'subscription': subscription,
'service_results': results,
'cripts_type': cripts_type,
'identifier': identifier,
'service_list': service_list},
RequestContext(request))
return HttpResponse(json.dumps(response), content_type="application/json")
def add_log(object_type, object_id, analysis_id, log_message, level, analyst):
"""
Add a log entry to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param log_message: The log entry to append.
:type log_message: dict
:param level: The log level.
:type level: str
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
results['message'] = "Could not find object to add results to."
return results
# Update analysis log
le = EmbeddedAnalysisResultLog()
le.message = log_message
le.level = level
le.datetime = str(datetime.datetime.now())
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(push__log=le)
results['success'] = True
else:
results['message'] = "Could not find task to add log to."
return results
def finish_task(object_type, object_id, analysis_id, status, analyst):
"""
Finish a task by setting its status to "completed" and setting the finish
date.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param status: The status of the task.
:type status: str ("error", "completed")
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not status:
status = "completed"
if status not in ('error', 'completed'):
status = "completed"
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
results['message'] = "Could not find object to add results to."
return results
# Update analysis log
date = str(datetime.datetime.now())
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(set__status=status,
set__finish_date=date)
results['success'] = True
return results
def create_notification(obj, username, message, source_filter=None,
notification_type=NotificationType.ALERT):
"""
Generate a notification -- based on mongo obj.
:param obj: The object.
:type obj: class which inherits from
:class:`cripts.core.cripts_mongoengine.CriptsBaseAttributes`
:param username: The user creating the notification.
:type username: str
:param message: The notification message.
:type message: str
:param source_filter: Filter on who can see this notification.
:type source_filter: list(str)
:param notification_type: The notification type (e.g. alert, error).
:type notification_type: str
"""
n = Notification()
n.analyst = username
obj_type = obj._meta['cripts_type']
users = set()
if notification_type not in NotificationType.ALL:
notification_type = NotificationType.ALERT
n.notification_type = notification_type
if obj_type == 'Comment':
n.obj_id = obj.obj_id
n.obj_type = obj.obj_type
n.notification = "%s added a comment: %s" % (username, obj.comment)
users.update(obj.users) # notify mentioned users
# for comments, use the sources from the object that it is linked to
# instead of the comments's sources
obj = class_from_id(n.obj_type, n.obj_id)
else:
n.notification = message
n.obj_id = obj.id
n.obj_type = obj_type
if hasattr(obj, 'source'):
sources = [s.name for s in obj.source]
subscribed_users = get_subscribed_users(n.obj_type, n.obj_id, sources)
# Filter on users that have access to the source of the object
for subscribed_user in subscribed_users:
allowed_sources = user_sources(subscribed_user)
for allowed_source in allowed_sources:
if allowed_source in sources:
if source_filter is None or allowed_source in source_filter:
users.add(subscribed_user)
break
else:
users.update(get_subscribed_users(n.obj_type, n.obj_id, []))
users.discard(username) # don't notify the user creating this notification
n.users = list(users)
if not len(n.users):
return
try:
n.save()
except ValidationError:
pass
# Signal potentially waiting threads that notification information is available
for user in n.users:
notification_lock = NotificationLockManager.get_notification_lock(user)
notification_lock.acquire()
try:
notification_lock.notifyAll()
finally:
notification_lock.release()
def get_email_address_details(address, analyst):
"""
Generate the data to render the Email Address details template.
:param address: The name of the Address to get details for.
:type address: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
address_object = EmailAddress.objects(address=address,
source__name__in=allowed_sources).first()
if not address_object:
error = ("Either no data exists for this email address"
" or you do not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
address_object.sanitize_sources(username="******" % analyst,
sources=allowed_sources)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, address_object.id, 'EmailAddress')
# subscription
subscription = {
'type': 'EmailAddress',
'id': address_object.id,
'subscribed': is_user_subscribed("%s" % analyst,
'EmailAddress',
address_object.id),
}
#objects
objects = address_object.sort_objects()
#relationships
relationships = address_object.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {
'type': 'EmailAddress',
'value': address_object.id
}
#comments
comments = {'comments': address_object.get_comments(),
'url_key':address_object.address}
# favorites
favorite = is_user_favorite("%s" % analyst, 'EmailAddress', address_object.id)
# services
service_list = get_supported_services('EmailAddress')
# analysis results
service_results = address_object.get_analysis_results()
args = {'email_address': address_object,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'address': address_object.address,
'service_list': service_list,
'service_results': service_results}
return template, args
def get_event_details(event_id, analyst):
"""
Generate the data to render the Event details template.
:param event_id: The ObjectId of the Event to get details for.
:type event_id: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
event = Event.objects(id=event_id, source__name__in=sources).first()
if not event:
template = "error.html"
args = {'error': "ID does not exist or insufficient privs for source"}
return template, args
event.sanitize("%s" % analyst)
download_form = DownloadFileForm(initial={
"obj_type": 'Event',
"obj_id": event_id
})
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, event.id, 'Event')
# subscription
subscription = {
'type': 'Event',
'id': event.id,
'subscribed': is_user_subscribed("%s" % analyst, 'Event', event.id),
}
#objects
objects = event.sort_objects()
#relationships
relationships = event.sort_relationships("%s" % analyst, meta=True)
# Get count of related Events for each related Sample
for smp in relationships.get('Sample', []):
count = Event.objects(relationships__object_id=smp['id'],
source__name__in=sources).count()
smp['rel_smp_events'] = count
# relationship
relationship = {'type': 'Event', 'value': event.id}
#comments
comments = {'comments': event.get_comments(), 'url_key': event.id}
# favorites
favorite = is_user_favorite("%s" % analyst, 'Event', event.id)
# services
service_list = get_supported_services('Event')
# analysis results
service_results = event.get_analysis_results()
args = {
'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'event': event,
'service_results': service_results,
'download_form': download_form
}
return template, args
def add_sample_for_event(event_id,
data,
analyst,
filedata=None,
filename=None,
md5=None,
email_addr=None,
inherit_sources=False):
"""
Add a sample related to this Event.
:param event_id: The ObjectId of the Event to associate with.
:type event_id: str
:param data: The form data.
:type data: dict
:param analyst: The user adding this Sample.
:type analyst: str
:param filedata: The sample data.
:type filedata: file handle.
:param filename: The name of the file.
:type filename: str
:param md5: The MD5 of the file.
:type md5: str
:param email_addr: Email address to which to email the sample
:type email_addr: str
:param inherit_sources: 'True' if Sample should inherit Event's Source(s)
:type inherit_sources: bool
:returns: dict with keys "success" (boolean) and "message" (str)
"""
response = {
'success': False,
'message': 'Unknown error; unable to upload file.'
}
users_sources = user_sources(analyst)
event = Event.objects(id=event_id, source__name__in=users_sources).first()
if not event:
return {'success': False, 'message': "No matching event found"}
source = data['source']
reference = data['reference']
file_format = data['file_format']
bucket_list = data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
ticket = data[form_consts.Common.TICKET_VARIABLE_NAME]
method = data['method']
if filename:
filename = filename.strip()
inherited_source = event.source if inherit_sources else None
try:
if filedata:
result = handle_uploaded_file(filedata,
source,
method,
reference,
file_format,
data['password'],
analyst,
related_id=event.id,
related_type='Event',
filename=filename,
bucket_list=bucket_list,
ticket=ticket,
inherited_source=inherited_source)
else:
result = handle_uploaded_file(None,
source,
method,
reference,
file_format,
None,
analyst,
related_id=event.id,
related_type='Event',
filename=filename,
md5=md5,
bucket_list=bucket_list,
ticket=ticket,
inherited_source=inherited_source,
is_return_only_md5=False)
except ZipFileError, zfe:
return {'success': False, 'message': zfe.value}
def get_email_address_details(address, analyst):
"""
Generate the data to render the Email Address details template.
:param address: The name of the Address to get details for.
:type address: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
address_object = EmailAddress.objects(
address=address, source__name__in=allowed_sources).first()
if not address_object:
error = ("Either no data exists for this email address"
" or you do not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
address_object.sanitize_sources(username="******" % analyst,
sources=allowed_sources)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, address_object.id,
'EmailAddress')
# subscription
subscription = {
'type':
'EmailAddress',
'id':
address_object.id,
'subscribed':
is_user_subscribed("%s" % analyst, 'EmailAddress', address_object.id),
}
#objects
objects = address_object.sort_objects()
#relationships
relationships = address_object.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'EmailAddress', 'value': address_object.id}
#comments
comments = {
'comments': address_object.get_comments(),
'url_key': address_object.address
}
# favorites
favorite = is_user_favorite("%s" % analyst, 'EmailAddress',
address_object.id)
# services
service_list = get_supported_services('EmailAddress')
# analysis results
service_results = address_object.get_analysis_results()
args = {
'email_address': address_object,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'address': address_object.address,
'service_list': service_list,
'service_results': service_results
}
return template, args
def get_event_details(event_id, analyst):
"""
Generate the data to render the Event details template.
:param event_id: The ObjectId of the Event to get details for.
:type event_id: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
event = Event.objects(id=event_id, source__name__in=sources).first()
if not event:
template = "error.html"
args = {'error': "ID does not exist or insufficient privs for source"}
return template, args
event.sanitize("%s" % analyst)
download_form = DownloadFileForm(initial={"obj_type": 'Event',
"obj_id": event_id})
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, event.id, 'Event')
# subscription
subscription = {
'type': 'Event',
'id': event.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Event', event.id),
}
#objects
objects = event.sort_objects()
#relationships
relationships = event.sort_relationships("%s" % analyst, meta=True)
# Get count of related Events for each related Sample
for smp in relationships.get('Sample', []):
count = Event.objects(relationships__object_id=smp['id'],
source__name__in=sources).count()
smp['rel_smp_events'] = count
# relationship
relationship = {
'type': 'Event',
'value': event.id
}
#comments
comments = {'comments': event.get_comments(), 'url_key': event.id}
# favorites
favorite = is_user_favorite("%s" % analyst, 'Event', event.id)
# services
service_list = get_supported_services('Event')
# analysis results
service_results = event.get_analysis_results()
args = {'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'event': event,
'service_results': service_results,
'download_form': download_form}
return template, args
def add_sample_for_event(event_id, data, analyst, filedata=None, filename=None,
md5=None, email_addr=None, inherit_sources=False):
"""
Add a sample related to this Event.
:param event_id: The ObjectId of the Event to associate with.
:type event_id: str
:param data: The form data.
:type data: dict
:param analyst: The user adding this Sample.
:type analyst: str
:param filedata: The sample data.
:type filedata: file handle.
:param filename: The name of the file.
:type filename: str
:param md5: The MD5 of the file.
:type md5: str
:param email_addr: Email address to which to email the sample
:type email_addr: str
:param inherit_sources: 'True' if Sample should inherit Event's Source(s)
:type inherit_sources: bool
:returns: dict with keys "success" (boolean) and "message" (str)
"""
response = {'success': False,
'message': 'Unknown error; unable to upload file.'}
users_sources = user_sources(analyst)
event = Event.objects(id=event_id, source__name__in=users_sources).first()
if not event:
return {'success': False,
'message': "No matching event found"}
source = data['source']
reference = data['reference']
file_format = data['file_format']
bucket_list = data[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
ticket = data[form_consts.Common.TICKET_VARIABLE_NAME]
method = data['method']
if filename:
filename = filename.strip()
inherited_source = event.source if inherit_sources else None
try:
if filedata:
result = handle_uploaded_file(filedata,
source,
method,
reference,
file_format,
data['password'],
analyst,
related_id=event.id,
related_type='Event',
filename=filename,
bucket_list=bucket_list,
ticket=ticket,
inherited_source=inherited_source)
else:
result = handle_uploaded_file(None,
source,
method,
reference,
file_format,
None,
analyst,
related_id=event.id,
related_type='Event',
filename=filename,
md5=md5,
bucket_list=bucket_list,
ticket=ticket,
inherited_source=inherited_source,
is_return_only_md5=False)
except ZipFileError, zfe:
return {'success': False, 'message': zfe.value}
def create_notification(obj,
username,
message,
source_filter=None,
notification_type=NotificationType.ALERT):
"""
Generate a notification -- based on mongo obj.
:param obj: The object.
:type obj: class which inherits from
:class:`cripts.core.cripts_mongoengine.CriptsBaseAttributes`
:param username: The user creating the notification.
:type username: str
:param message: The notification message.
:type message: str
:param source_filter: Filter on who can see this notification.
:type source_filter: list(str)
:param notification_type: The notification type (e.g. alert, error).
:type notification_type: str
"""
n = Notification()
n.analyst = username
obj_type = obj._meta['cripts_type']
users = set()
if notification_type not in NotificationType.ALL:
notification_type = NotificationType.ALERT
n.notification_type = notification_type
if obj_type == 'Comment':
n.obj_id = obj.obj_id
n.obj_type = obj.obj_type
n.notification = "%s added a comment: %s" % (username, obj.comment)
users.update(obj.users) # notify mentioned users
# for comments, use the sources from the object that it is linked to
# instead of the comments's sources
obj = class_from_id(n.obj_type, n.obj_id)
else:
n.notification = message
n.obj_id = obj.id
n.obj_type = obj_type
if hasattr(obj, 'source'):
sources = [s.name for s in obj.source]
subscribed_users = get_subscribed_users(n.obj_type, n.obj_id, sources)
# Filter on users that have access to the source of the object
for subscribed_user in subscribed_users:
allowed_sources = user_sources(subscribed_user)
for allowed_source in allowed_sources:
if allowed_source in sources:
if source_filter is None or allowed_source in source_filter:
users.add(subscribed_user)
break
else:
users.update(get_subscribed_users(n.obj_type, n.obj_id, []))
users.discard(username) # don't notify the user creating this notification
n.users = list(users)
if not len(n.users):
return
try:
n.save()
except ValidationError:
pass
# Signal potentially waiting threads that notification information is available
for user in n.users:
notification_lock = NotificationLockManager.get_notification_lock(user)
notification_lock.acquire()
try:
notification_lock.notifyAll()
finally:
notification_lock.release()