def target_user_stats():
"""
Generate targets from email To/CC fields, then generate divisions from
targets list.
No cleanup or logic is being done on the To/CC fields. If they are not
valid email addresses (user@domain), they do not get added as a target.
"""
mapcode = """
function () {
try {
this.to.forEach(function(z) {
emit(z.toLowerCase(), {count: 1});
});
} catch(err) {}
}
"""
reducecode = """
function(k,v) {
var count = 0;
v.forEach(function(v) {
count += v["count"];
});
return {count: count};
}
"""
m = Code(mapcode)
r = Code(reducecode)
results = Email.objects(to__exists=True).map_reduce(m, r, 'inline')
for result in results:
try:
targs = Target.objects(email_address__iexact=result.key)
if not targs:
targs = [Target()]
targs[0].email_address = result.key.strip().lower()
for targ in targs:
targ.email_count = result.value['count']
targ.save()
except:
pass
mapcode = """
function() {
if ("division" in this) {
emit(this.division, {count: this.email_count})
}
}
"""
m = Code(mapcode)
try:
results = Target.objects().map_reduce(m, r, 'inline')
for result in results:
div = Division.objects(division__iexact=result.key).first()
if not div:
div = Division()
div.division = result.key
div.email_count = result.value['count']
div.save()
except:
raise
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
# Get addresses from the 'to' field of emails attributed to this campaign
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
addresses[to.strip().lower()] = 1 # add the way it should be
addresses[to] = 1 # also add the way it is in the Email
# Get addresses of Targets attributed to this campaign
targets = Target.objects(campaign__name=campaign).only('email_address')
for target in targets:
addresses[target.email_address] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def get_campaign_targets(campaign,user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def get_campaign_targets(campaign, user):
"""
Get targets related to a specific campaign.
:param campaign: The campaign to search for.
:type campaign: str
:param user: The user requesting this information.
:type user: str
:returns: list
"""
# Searching for campaign targets
sourcefilt = user_sources(user)
# Get addresses from the 'to' field of emails attributed to this campaign
emails = Email.objects(source__name__in=sourcefilt,
campaign__name=campaign).only('to')
addresses = {}
for email in emails:
for to in email['to']:
addresses[to.strip().lower()] = 1 # add the way it should be
addresses[to] = 1 # also add the way it is in the Email
# Get addresses of Targets attributed to this campaign
targets = Target.objects(campaign__name=campaign).only('email_address')
for target in targets:
addresses[target.email_address] = 1
uniq_addrs = addresses.keys()
return uniq_addrs
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def get_target(email_address=None):
"""
Get a target from the database.
:param email_address: The email address of the target to get.
:type email_address: str
:returns: None, :class:`crits.targets.target.Target`
"""
if not email_address:
return None
target = Target.objects(email_address=email_address).first()
return target
def get_target(email_address=None):
"""
Get a target from the database.
:param email_address: The email address of the target to get.
:type email_address: str
:returns: None, :class:`crits.targets.target.Target`
"""
if not email_address:
return None
target = Target.objects(email_address=email_address).first()
return target
def handle(self, *args, **options):
"""
Script Execution.
"""
self.is_delete = options.get('is_delete')
mapcode = """
function () {
try {
this.to.forEach(function(z) {
emit(z.toLowerCase(), {count: 1});
});
} catch(err) {}
}
"""
reducecode = """
function(k,v) {
var count = 0;
v.forEach(function(v) {
count += v["count"];
});
return {count: count};
}
"""
m = Code(mapcode)
r = Code(reducecode)
results = Email.objects(to__exists=True).map_reduce(m, r, 'inline')
for result in results:
try:
targets = Target.objects(email_address__iexact=result.key)
targ_dup_count = targets.count()
if targ_dup_count > 1:
print str(result.key) + " [" + str(targ_dup_count) + "]"
for target in targets:
print target.to_json()
if self.is_delete:
delete_up_to = targets.count() - 1
for target in targets[:delete_up_to]:
print "Deleting target: " + str(target.id)
target.delete()
except Exception, e:
print e
pass
def handle(self, *args, **options):
"""
Script Execution.
"""
self.is_delete = options.get('is_delete')
mapcode = """
function () {
try {
this.to.forEach(function(z) {
emit(z.toLowerCase(), {count: 1});
});
} catch(err) {}
}
"""
reducecode = """
function(k,v) {
var count = 0;
v.forEach(function(v) {
count += v["count"];
});
return {count: count};
}
"""
m = Code(mapcode)
r = Code(reducecode)
results = Email.objects(to__exists=True).map_reduce(m, r, 'inline')
for result in results:
try:
targets = Target.objects(email_address__iexact=result.key)
targ_dup_count = targets.count()
if targ_dup_count > 1:
print str(result.key) + " [" + str(targ_dup_count) + "]"
for target in targets:
print target.to_json()
if self.is_delete:
delete_up_to = targets.count() - 1
for target in targets[:delete_up_to]:
print "Deleting target: " + str(target.id)
target.delete()
except Exception, e:
print e
pass
def remove_target(email_address=None, analyst=None):
"""
Remove a target.
:param email_address: The email address of the target to remove.
:type email_address: str
:param analyst: The user removing the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if not email_address:
return {"success": False, "message": "No email address to look up"}
target = Target.objects(email_address=email_address).first()
if not target:
return {"success": False, "message": "No target matching this email address."}
target.delete(username=analyst)
return {"success": True, "message": "Target removed successfully"}
def remove_target(email_address=None, analyst=None):
"""
Remove a target.
:param email_address: The email address of the target to remove.
:type email_address: str
:param analyst: The user removing the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if not email_address:
return {'success': False, 'message': "No email address to look up"}
target = Target.objects(email_address=email_address).first()
if not target:
return {
'success': False,
'message': "No target matching this email address."
}
target.delete(username=analyst)
return {'success': True, 'message': "Target removed successfully"}
def get_target_details(email_address, analyst):
"""
Generate the data to render the Target details template.
:param email_address: The email address of the target.
:type email_address: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
if not email_address:
template = "error.html"
args = {'error': "Must provide an email address."}
return template, args
# check for exact match first
target = Target.objects(email_address=email_address).first()
if not target: # if no exact match, look for case-insensitive match
target = Target.objects(email_address__iexact=email_address).first()
if not target:
target = Target()
target.email_address = email_address.strip().lower()
form = TargetInfoForm(initial={'email_address': email_address})
email_list = target.find_emails(analyst)
form = TargetInfoForm(initial=target.to_dict())
if form.fields.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) != None:
form.fields.pop(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if form.fields.get(form_consts.Common.TICKET_VARIABLE_NAME) != None:
form.fields.pop(form_consts.Common.TICKET_VARIABLE_NAME)
subscription = {
'type': 'Target',
'id': target.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Target',
target.id)
}
#objects
objects = target.sort_objects()
#relationships
relationships = target.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {
'type': 'Target',
'value': target.id
}
#comments
if target.id:
comments = {'comments': target.get_comments(),
'url_key': email_address}
else:
comments = {'comments': [],
'url_key': email_address}
#screenshots
screenshots = target.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Target', target.id)
# analysis results
service_results = target.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
'relationship': relationship,
'comments': comments,
'favorite': favorite,
'subscription': subscription,
'screenshots': screenshots,
'email_list': email_list,
'target_detail': target,
'service_results': service_results,
'form': form}
return template, args
def get_target_details(email_address, analyst):
"""
Generate the data to render the Target details template.
:param email_address: The email address of the target.
:type email_address: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
if not email_address:
template = "error.html"
args = {'error': "Must provide an email address."}
return template, args
# check for exact match first
target = Target.objects(email_address=email_address).first()
if not target: # if no exact match, look for case-insensitive match
target = Target.objects(email_address__iexact=email_address).first()
if not target:
target = Target()
target.email_address = email_address.strip().lower()
form = TargetInfoForm(initial={'email_address': email_address})
email_list = target.find_emails(analyst)
form = TargetInfoForm(initial=target.to_dict())
if form.fields.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) != None:
form.fields.pop(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if form.fields.get(form_consts.Common.TICKET_VARIABLE_NAME) != None:
form.fields.pop(form_consts.Common.TICKET_VARIABLE_NAME)
subscription = {
'type': 'Target',
'id': target.id,
'subscribed': is_user_subscribed("%s" % analyst, 'Target', target.id)
}
#objects
objects = target.sort_objects()
#relationships
relationships = target.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'Target', 'value': target.id}
#comments
if target.id:
comments = {
'comments': target.get_comments(),
'url_key': email_address
}
else:
comments = {'comments': [], 'url_key': email_address}
#screenshots
screenshots = target.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Target', target.id)
# analysis results
service_results = target.get_analysis_results()
args = {
'objects': objects,
'relationships': relationships,
'relationship': relationship,
'comments': comments,
'favorite': favorite,
'subscription': subscription,
'screenshots': screenshots,
'email_list': email_list,
'target_detail': target,
'service_results': service_results,
'form': form
}
return template, args
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id, 'Campaign')
# subscription
subscription = {
'type': 'Campaign',
'id': campaign_detail.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Campaign',
campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {'comments': campaign_detail.get_comments(),
'url_key': campaign_name}
#screenshots
screenshots = campaign_detail.get_screenshots(analyst)
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(source__name__in=sources,
__raw__=formatted_query).count()
# Item counts for targets
emails = Email.objects(source__name__in=sources, __raw__=formatted_query)
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
# analysis results
service_results = campaign_detail.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"screenshots": screenshots,
'service_results': service_results,
"ttp_form": ttp_form}
return template, args
def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if 'email_address' not in data:
return {'success': False, 'message': "No email address to look up"}
target = Target.objects(email_address=data['email_address']).first()
if not target:
target = Target()
target.email_address = data['email_address']
target.department = data['department']
target.division = data['division']
target.organization_id = data['organization_id']
target.firstname = data['firstname']
target.lastname = data['lastname']
target.note = data['note']
target.title = data['title']
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
try:
target.save(username=analyst)
target.reload()
return {
'success': True,
'message': "Target saved successfully",
'id': str(target.id)
}
except ValidationError, e:
return {'success': False, 'message': "Target save failed: %s" % e}
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in [
'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator',
'Screenshot'
] and not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(message_id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Signature':
return Signature.objects(md5=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'Action':
return Action.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'Signature':
return Signature.objects(id=_id).first()
elif type_ == 'SignatureType':
return SignatureType.objects(id=_id).first()
elif type_ == 'SignatureDependency':
return SignatureDependency.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
'Indicator', 'Screenshot'] and
not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def get_target_details(email_address, analyst):
"""
Generate the data to render the Target details template.
:param email_address: The email address of the target.
:type email_address: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
if not email_address:
template = "error.html"
args = {"error": "Must provide an email address."}
return template, args
target = Target.objects(email_address=email_address).first()
if not target:
target = Target()
target.email_address = email_address
form = TargetInfoForm(initial={"email_address": email_address})
email_list = target.find_emails(analyst)
# initial_data = target.to_dict()
# initial_data['bucket_list'] = target.get_bucket_list_string();
form = TargetInfoForm(initial=target.to_dict())
if form.fields.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME) != None:
form.fields.pop(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if form.fields.get(form_consts.Common.TICKET_VARIABLE_NAME) != None:
form.fields.pop(form_consts.Common.TICKET_VARIABLE_NAME)
subscription = {
"type": "Target",
"id": target.id,
"subscribed": is_user_subscribed("%s" % analyst, "Target", target.id),
}
# objects
objects = target.sort_objects()
# relationships
relationships = target.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {"type": "Target", "value": target.id}
# comments
if target.id:
comments = {"comments": target.get_comments(), "url_key": email_address}
else:
comments = {"comments": [], "url_key": email_address}
# screenshots
screenshots = target.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, "Target", target.id)
args = {
"objects": objects,
"relationships": relationships,
"relationship": relationship,
"comments": comments,
"favorite": favorite,
"subscription": subscription,
"screenshots": screenshots,
"email_list": email_list,
"target_detail": target,
"form": form,
}
return template, args
def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if 'email_address' not in data:
return {'success': False, 'message': "No email address to look up"}
# check for exact match first
target = Target.objects(email_address=data['email_address']).first()
if not target: # if no exact match, look for case-insensitive match
target = Target.objects(
email_address__iexact=data['email_address']).first()
is_new = False
if not target:
is_new = True
target = Target()
target.email_address = data['email_address'].strip().lower()
bucket_list = False
ticket = False
if 'department' in data:
target.department = data['department']
if 'division' in data:
target.division = data['division']
if 'organization_id' in data:
target.organization_id = data['organization_id']
if 'firstname' in data:
target.firstname = data['firstname']
if 'lastname' in data:
target.lastname = data['lastname']
if 'note' in data:
target.note = data['note']
if 'title' in data:
target.title = data['title']
if 'campaign' in data and 'camp_conf' in data:
target.add_campaign(
EmbeddedCampaign(name=data['campaign'],
confidence=data['camp_conf'],
analyst=analyst))
if 'bucket_list' in data:
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if 'ticket' in data:
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
try:
target.save(username=analyst)
target.reload()
if is_new:
run_triage(target, analyst)
return {
'success': True,
'message': "Target saved successfully",
'id': str(target.id)
}
except ValidationError, e:
return {'success': False, 'message': "Target save failed: %s" % e}
def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if "email_address" not in data:
return {"success": False, "message": "No email address to look up"}
target = Target.objects(email_address=data["email_address"]).first()
if not target:
target = Target()
target.email_address = data["email_address"]
target.department = data["department"]
target.division = data["division"]
target.organization_id = data["organization_id"]
target.firstname = data["firstname"]
target.lastname = data["lastname"]
target.note = data["note"]
target.title = data["title"]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
try:
target.save(username=analyst)
target.reload()
return {"success": True, "message": "Target saved successfully", "id": str(target.id)}
except ValidationError, e:
return {"success": False, "message": "Target save failed: %s" % e}
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
campaign_detail.sanitize(username=analyst)
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id, 'Campaign')
# subscription
subscription = {
'type': 'Campaign',
'id': campaign_detail.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Campaign',
campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {'comments': campaign_detail.get_comments(),
'url_key': campaign_name}
#screenshots
screenshots = campaign_detail.get_screenshots(analyst)
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Actor, Backdoor, Exploit, Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(source__name__in=sources,
__raw__=formatted_query).count()
# Item counts for targets
uniq_addrs = get_campaign_targets(campaign_name, analyst)
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
# analysis results
service_results = campaign_detail.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"screenshots": screenshots,
'service_results': service_results,
"ttp_form": ttp_form,
"CampaignACL": CampaignACL}
return template, args
def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if 'email_address' not in data:
return {'success': False,
'message': "No email address to look up"}
# check for exact match first
target = Target.objects(email_address=data['email_address']).first()
if not target: # if no exact match, look for case-insensitive match
target = Target.objects(email_address__iexact=data['email_address']).first()
is_new = False
if not target:
is_new = True
target = Target()
target.email_address = data['email_address'].strip().lower()
bucket_list = False
ticket = False
if 'department' in data:
target.department = data['department']
if 'division' in data:
target.division = data['division']
if 'organization_id' in data:
target.organization_id = data['organization_id']
if 'firstname' in data:
target.firstname = data['firstname']
if 'lastname' in data:
target.lastname = data['lastname']
if 'note' in data:
target.note = data['note']
if 'title' in data:
target.title = data['title']
if 'campaign' in data and 'camp_conf' in data:
target.add_campaign(EmbeddedCampaign(name=data['campaign'],
confidence=data['camp_conf'],
analyst=analyst))
if 'bucket_list' in data:
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if 'ticket' in data:
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if 'related_id' in data:
related_id = data['related_id']
if 'related_type' in data:
related_type = data['related_type']
if 'relationship_type' in data:
relationship_type = data['relationship_type']
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
target.save(username=analyst)
if related_obj and target:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
target.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
target.save(username=analyst)
target.reload()
if is_new:
run_triage(target, analyst)
return {'success': True,
'message': "Target saved successfully",
'id': str(target.id)}
except ValidationError, e:
return {'success': False,
'message': "Target save failed: %s" % e}
def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if 'email_address' not in data:
return {'success': False,
'message': "No email address to look up"}
target = Target.objects(email_address=data['email_address']).first()
is_new = False
if not target:
is_new = True
target = Target()
target.email_address = data['email_address']
target.department = data['department']
target.division = data['division']
target.organization_id = data['organization_id']
target.firstname = data['firstname']
target.lastname = data['lastname']
target.note = data['note']
target.title = data['title']
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
try:
target.save(username=analyst)
target.reload()
if is_new:
run_triage(target, analyst)
return {'success': True,
'message': "Target saved successfully",
'id': str(target.id)}
except ValidationError, e:
return {'success': False,
'message': "Target save failed: %s" % e}
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id,
'Campaign')
# subscription
subscription = {
'type':
'Campaign',
'id':
campaign_detail.id,
'subscribed':
is_user_subscribed("%s" % analyst, 'Campaign', campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {
'comments': campaign_detail.get_comments(),
'url_key': campaign_name
}
#screenshots
screenshots = campaign_detail.get_screenshots(analyst)
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(
source__name__in=sources, __raw__=formatted_query).count()
# Item counts for targets
uniq_addrs = get_campaign_targets(campaign_name, analyst)
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
# analysis results
service_results = campaign_detail.get_analysis_results()
args = {
'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"screenshots": screenshots,
'service_results': service_results,
"ttp_form": ttp_form
}
return template, args
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# Quick fail
if not _id or not type_:
return None
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode("utf8")):
return None
if type_ == "Actor":
return Actor.objects(id=_id).first()
elif type_ == "Backdoor":
return Backdoor.objects(id=_id).first()
elif type_ == "ActorThreatIdentifier":
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == "Campaign":
return Campaign.objects(id=_id).first()
elif type_ == "Certificate":
return Certificate.objects(id=_id).first()
elif type_ == "Comment":
return Comment.objects(id=_id).first()
elif type_ == "Domain":
return Domain.objects(id=_id).first()
elif type_ == "Email":
return Email.objects(id=_id).first()
elif type_ == "Event":
return Event.objects(id=_id).first()
elif type_ == "Exploit":
return Exploit.objects(id=_id).first()
elif type_ == "Indicator":
return Indicator.objects(id=_id).first()
elif type_ == "Action":
return Action.objects(id=_id).first()
elif type_ == "IP":
return IP.objects(id=_id).first()
elif type_ == "PCAP":
return PCAP.objects(id=_id).first()
elif type_ == "RawData":
return RawData.objects(id=_id).first()
elif type_ == "RawDataType":
return RawDataType.objects(id=_id).first()
elif type_ == "Sample":
return Sample.objects(id=_id).first()
elif type_ == "Signature":
return Signature.objects(id=_id).first()
elif type_ == "SignatureType":
return SignatureType.objects(id=_id).first()
elif type_ == "SignatureDependency":
return SignatureDependency.objects(id=_id).first()
elif type_ == "SourceAccess":
return SourceAccess.objects(id=_id).first()
elif type_ == "Screenshot":
return Screenshot.objects(id=_id).first()
elif type_ == "Target":
return Target.objects(id=_id).first()
elif type_ == "UserRole":
return UserRole.objects(id=_id).first()
else:
return None
