def setUp(self):
prep_db()
self.user = CRITsUser.objects(username=TUSER_NAME).first()
self.user2 = CRITsUser.objects(username=TUSER2_NAME).first()
self.campaign1 = Campaign.objects(name=TCAMPAIGN1).first()
self.campaign2 = Campaign.objects(name=TCAMPAIGN2).first()
forge_relationship(left_class=self.campaign1,
right_class=self.campaign2,
rel_type=TRELATIONSHIP_TYPE,
analyst=self.user.username,
rel_confidence=TRELATIONSHIP_CONFIDENCE)
def add_ttp(cid, ttp, analyst):
"""
Add a TTP to a Campaign.
:param cid: ObjectId of the Campaign.
:type cid: str
:param ttp: The TTP to add.
:type ttp: str
:param analyst: The user adding the TTP.
:type analyst: str
:returns: dict with keys:
'success' (boolean),
'campaign' (:class:`crits.campaigns.campaign.Campaign`) if success,
'message' (str) if failed.
"""
campaign = Campaign.objects(id=cid).first()
if campaign:
new_ttp = EmbeddedTTP()
new_ttp.analyst = analyst
new_ttp.ttp = ttp
try:
campaign.add_ttp(new_ttp)
campaign.save(username=analyst)
return {'success': True, 'campaign': campaign}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def clean_db():
"""
Clean database for test.
"""
user = CRITsUser.objects(username=TUSER_NAME).first()
if user:
user.delete()
user2 = CRITsUser.objects(username=TUSER2_NAME).first()
if user2:
user2.delete()
campaign1 = Campaign.objects(name=TCAMPAIGN1).first()
if campaign1:
campaign1.delete()
campaign2 = Campaign.objects(name=TCAMPAIGN2).first()
if campaign2:
campaign2.delete()
def add_campaign_from_nodes(name, confidence, nodes, user):
result = { "success": False }
# Make sure Campaign exists
campaign_obj = Campaign.objects(name=name).first()
if not campaign_obj:
result["message"] = "Invalid campaign name."
return result
campaign = EmbeddedCampaign(name=name, confidence=confidence, analyst=user)
counter = 0
for node in nodes:
id_ = node.get('id', None)
type_ = node.get('type', None)
# Must have type and id, and type must not be Campaign
if not id_ or not type_ or type_.lower() == 'campaign':
continue
obj = class_from_id(type_, id_)
if not obj:
continue
obj.add_campaign(campaign)
obj.save()
counter += 1
result["message"] = "%s nodes processed" % counter
result["success"] = True
return result
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def add_campaign(name, description, aliases, analyst, bucket_list=None,
ticket=None):
"""
Add a Campaign.
:param name: The name of the new Campaign.
:type name: str
:param description: Description of the new Campaign.
:type description: str
:param aliases: Aliases for the new Campaign.
:type aliases: str (comma separated) or list.
:param analyst: The user adding the Campaign.
:type analyst: str
:param bucket_list: Buckets to add to this Campaign.
:type bucket_list: str (comma separated) or list.
:param ticket: Ticket(s) to add to this Campaign.
:type ticket: str (comma separated) or list.
:returns: dict with key 'success' (boolean) and 'message' (str).
"""
# Verify the Campaign does not exist.
campaign = Campaign.objects(name=name).first()
if campaign:
return {'success': False, 'message': ['Campaign already exists.'],
'id': str(campaign.id)}
# Create new campaign.
campaign = Campaign(name=name)
campaign.edit_description(description)
if bucket_list:
campaign.add_bucket_list(bucket_list, analyst)
if ticket:
campaign.add_ticket(ticket, analyst)
# Adjust aliases.
if isinstance(aliases, basestring):
alias_list = aliases.split(',')
final_aliases = [a.strip() for a in alias_list]
elif isinstance(aliases, list):
final_aliases = [a.strip() for a in aliases]
else:
final_aliases = []
campaign.add_alias(final_aliases)
try:
campaign.save(username=analyst)
campaign.reload()
return {'success': True,
'message': 'Campaign created successfully!',
'id': str(campaign.id)}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def remove_campaign(name, analyst):
"""
Remove a Campaign.
:param name: The name of the Campaign to remove.
:type name: str
:param analyst: The user removing the Campaign.
:type analyst: str
:returns: dict with key 'success' (boolean) and 'message' (str) if failed.
"""
campaign = Campaign.objects(name=name).first()
if campaign:
campaign.delete(username=analyst)
return {'success': True}
else:
return {'success': False, 'message': 'Campaign not found.'}
def deactivate_campaign(name, analyst):
"""
Deactivate a Campaign.
:param name: Name of the Campaign.
:type name: str
:param analyst: The user deactivating the Campaign.
:type analyst: str
:returns: dict with key 'success' (boolean) and 'message' (str) if failed.
"""
campaign = Campaign.objects(name=name).first()
if campaign:
campaign.deactivate()
try:
campaign.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def modify_campaign_aliases(name, tags, analyst):
"""
Modify the aliases for a Campaign.
:param name: Name of the Campaign.
:type name: str
:param tags: The new aliases.
:type tags: list
:param analyst: The user setting the new aliases.
:type analyst: str
:returns: dict with key 'success' (boolean) and 'message' (str) if failed.
"""
campaign = Campaign.objects(name=name).first()
if campaign:
campaign.set_aliases(tags)
try:
campaign.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def update_campaign_description(cid, description, analyst):
"""
Update a Campaign description.
:param cid: ObjectId of the Campaign.
:type cid: str
:param description: The new description.
:type description: str
:param analyst: The user setting the new description.
:type analyst: str
:returns: dict with key 'success' (boolean) and 'message' (str) if failed.
"""
if not description:
return {'success': False, 'message': "No description to change"}
campaign = Campaign.objects(id=cid).first()
campaign.edit_description(description)
try:
campaign.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': e}
def edit_ttp(cid, old_ttp, new_ttp, analyst):
"""
Edit an existing TTP.
:param cid: ObjectId of the Campaign.
:type cid: str
:param old_ttp: Original value of the TTP.
:type old_ttp: str
:param new_ttp: New value of the TTP.
:type new_ttp: str
:param analyst: The user editing the TTP.
:type analyst: str
:returns: dict with key 'success' (boolean) and 'message' (str) if failed.
"""
campaign = Campaign.objects(id=cid).first()
if campaign:
try:
campaign.edit_ttp(old_ttp, new_ttp)
campaign.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def remove_ttp(cid, ttp, analyst):
"""
Remove a TTP from a Campaign.
:param cid: ObjectId of the Campaign.
:type cid: str
:param ttp: The TTP to remove.
:type ttp: str
:param analyst: The user removing the TTP.
:type analyst: str
:returns: dict with keys:
'success' (boolean),
'campaign' (:class:`crits.campaigns.campaign.Campaign`) if success,
'message' (str) if failed.
"""
campaign = Campaign.objects(id=cid).first()
if campaign:
try:
campaign.remove_ttp(ttp)
campaign.save(username=analyst)
return {'success': True, 'campaign': campaign}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def add_campaign(name,
description,
aliases,
analyst,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add a Campaign.
:param name: The name of the new Campaign.
:type name: str
:param description: Description of the new Campaign.
:type description: str
:param aliases: Aliases for the new Campaign.
:type aliases: str (comma separated) or list.
:param analyst: The user adding the Campaign.
:type analyst: str
:param bucket_list: Buckets to add to this Campaign.
:type bucket_list: str (comma separated) or list.
:param ticket: Ticket(s) to add to this Campaign.
:type ticket: str (comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with key 'success' (boolean) and 'message' (str).
"""
# Verify the Campaign does not exist.
campaign = Campaign.objects(name=name).first()
if campaign:
return {
'success': False,
'message': ['Campaign already exists.'],
'id': str(campaign.id)
}
# Create new campaign.
campaign = Campaign(name=name)
campaign.edit_description(description)
if bucket_list:
campaign.add_bucket_list(bucket_list, analyst)
if ticket:
campaign.add_ticket(ticket, analyst)
# Adjust aliases.
if isinstance(aliases, basestring):
alias_list = aliases.split(',')
final_aliases = [a.strip() for a in alias_list]
elif isinstance(aliases, list):
final_aliases = [a.strip() for a in aliases]
else:
final_aliases = []
campaign.add_alias(final_aliases)
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
campaign.save(username=analyst)
if related_obj and relationship_type and campaign:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
campaign.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
campaign.save(username=analyst)
campaign.reload()
try:
campaign.save(username=analyst)
campaign.reload()
return {
'success': True,
'message': 'Campaign created successfully!',
'id': str(campaign.id)
}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id, 'Campaign')
# subscription
subscription = {
'type': 'Campaign',
'id': campaign_detail.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Campaign',
campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {'comments': campaign_detail.get_comments(),
'url_key': campaign_name}
#screenshots
screenshots = campaign_detail.get_screenshots(analyst)
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(source__name__in=sources,
__raw__=formatted_query).count()
# Item counts for targets
emails = Email.objects(source__name__in=sources, __raw__=formatted_query)
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
# analysis results
service_results = campaign_detail.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"screenshots": screenshots,
'service_results': service_results,
"ttp_form": ttp_form}
return template, args
def handle_indicator_csv(csv_data, source, reference, ctype, username,
add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode('ascii')
data = csv.DictReader(StringIO(cdata), skipinitialspace=True)
result = {'success': True}
result_message = "Indicators added successfully!"
# Compute permitted values in CSV
valid_ratings = {
'unknown': 'unknown',
'benign': 'benign',
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
valid_ind_types = {}
for obj in ObjectType.objects(datatype__enum__exists=False, datatype__file__exists=False):
if obj['object_type'] == obj['name']:
name = obj['object_type']
else:
name = "%s - %s" % (obj['object_type'], obj['name'])
valid_ind_types[name.lower()] = name
# Start line-by-line import
processed = 0
for d in data:
processed += 1
ind = {}
ind['value'] = d.get('Indicator', '').lower().strip()
ind['type'] = get_verified_field(d, 'Type', valid_ind_types)
if not ind['value'] or not ind['type']:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result['success'] = False
if not ind['value']:
i += "No valid Indicator value. "
if not ind['type']:
i += "No valid Indicator type. "
result_message += "Cannot process row: %s. %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, 'Campaign', valid_campaigns)
if campaign:
ind['campaign'] = campaign
ind['campaign_confidence'] = get_verified_field(d, 'Campaign Confidence',
valid_campaign_confidence,
default='low')
ind['confidence'] = get_verified_field(d, 'Confidence', valid_ratings,
default='unknown')
ind['impact'] = get_verified_field(d, 'Impact', valid_ratings,
default='unknown')
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, '')
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, '')
try:
handle_indicator_insert(ind, source, reference, analyst=username,
add_domain=add_domain)
except Exception, e:
result['success'] = False
result['message'] = str(e)
return result
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
#Quick fail
if not _id or not type_:
return None
# doing this to avoid circular imports
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
from crits.actors.actor import Actor
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
from crits.backdoors.backdoor import Backdoor
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
from crits.actors.actor import ActorThreatIdentifier
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
from crits.campaigns.campaign import Campaign
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
from crits.certificates.certificate import Certificate
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
from crits.comments.comment import Comment
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
from crits.domains.domain import Domain
return Domain.objects(id=_id).first()
elif type_ == 'Email':
from crits.emails.email import Email
return Email.objects(id=_id).first()
elif type_ == 'Event':
from crits.events.event import Event
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
from crits.exploits.exploit import Exploit
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
from crits.indicators.indicator import Indicator
return Indicator.objects(id=_id).first()
elif type_ == 'Action':
from crits.core.crits_mongoengine import Action
return Action.objects(id=_id).first()
elif type_ == 'IP':
from crits.ips.ip import IP
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
from crits.pcaps.pcap import PCAP
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
from crits.raw_data.raw_data import RawData
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
from crits.raw_data.raw_data import RawDataType
return RawDataType.objects(id=_id).first()
elif type_ == 'Role':
from crits.core.role import Role
return Role.objects(id=_id).first()
elif type_ == 'Sample':
from crits.samples.sample import Sample
return Sample.objects(id=_id).first()
elif type_ == 'Signature':
from crits.signatures.signature import Signature
return Signature.objects(id=_id).first()
elif type_ == 'SignatureType':
from crits.signatures.signature import SignatureType
return SignatureType.objects(id=_id).first()
elif type_ == 'SignatureDependency':
from crits.signatures.signature import SignatureDependency
return SignatureDependency.objects(id=_id).first()
elif type_ == 'SourceAccess':
from crits.core.source_access import SourceAccess
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
from crits.screenshots.screenshot import Screenshot
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
from crits.targets.target import Target
return Target.objects(id=_id).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id,
'Campaign')
# subscription
subscription = {
'type':
'Campaign',
'id':
campaign_detail.id,
'subscribed':
is_user_subscribed("%s" % analyst, 'Campaign', campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {
'comments': campaign_detail.get_comments(),
'url_key': campaign_name
}
#screenshots
screenshots = campaign_detail.get_screenshots(analyst)
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(
source__name__in=sources, __raw__=formatted_query).count()
# Item counts for targets
emails = Email.objects(source__name__in=sources, __raw__=formatted_query)
addresses = {}
for email in emails:
for to in email['to']:
# This might be a slow operation since we're looking up all "to"
# targets, could possibly bulk search this.
target = Target.objects(email_address__iexact=to).first()
if target is not None:
addresses[target.email_address] = 1
else:
addresses[to] = 1
uniq_addrs = addresses.keys()
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
# analysis results
service_results = campaign_detail.get_analysis_results()
args = {
'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"screenshots": screenshots,
'service_results': service_results,
"ttp_form": ttp_form
}
return template, args
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatType, ActorMotivation
from crits.actors.actor import ActorSophistication, ActorIntendedEffect
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'ActorThreatType':
return ActorThreatType.objects(id=_id).first()
elif type_ == 'ActorMotivation':
return ActorMotivation.objects(id=_id).first()
elif type_ == 'ActorSophistication':
return ActorSophistication.objects(id=_id).first()
elif type_ == 'ActorIntendedEffect':
return ActorIntendedEffect.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def handle_indicator_csv(csv_data, source, method, reference, ctype, username, add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param method: The method of acquisition of this indicator.
:type method: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode("ascii")
data = csv.DictReader(BytesIO(cdata), skipinitialspace=True)
result = {"success": True}
result_message = ""
# Compute permitted values in CSV
valid_ratings = {"unknown": "unknown", "benign": "benign", "low": "low", "medium": "medium", "high": "high"}
valid_campaign_confidence = {"low": "low", "medium": "medium", "high": "high"}
valid_campaigns = {}
for c in Campaign.objects(active="on"):
valid_campaigns[c["name"].lower().replace(" - ", "-")] = c["name"]
valid_actions = {}
for a in Action.objects(active="on"):
valid_actions[a["name"].lower().replace(" - ", "-")] = a["name"]
valid_ind_types = {}
for obj in IndicatorTypes.values(sort=True):
valid_ind_types[obj.lower().replace(" - ", "-")] = obj
# Start line-by-line import
added = 0
for processed, d in enumerate(data, 1):
ind = {}
ind["value"] = d.get("Indicator", "").strip()
ind["lower"] = d.get("Indicator", "").lower().strip()
ind["description"] = d.get("Description", "").strip()
ind["type"] = get_verified_field(d, valid_ind_types, "Type")
ind["threat_type"] = d.get("Threat Type", IndicatorThreatTypes.UNKNOWN)
ind["attack_type"] = d.get("Attack Type", IndicatorAttackTypes.UNKNOWN)
if len(ind["threat_type"]) < 1:
ind["threat_type"] = IndicatorThreatTypes.UNKNOWN
if ind["threat_type"] not in IndicatorThreatTypes.values():
result["success"] = False
result_message += "Cannot process row %s: Invalid Threat Type<br />" % processed
continue
if len(ind["attack_type"]) < 1:
ind["attack_type"] = IndicatorAttackTypes.UNKNOWN
if ind["attack_type"] not in IndicatorAttackTypes.values():
result["success"] = False
result_message += "Cannot process row %s: Invalid Attack Type<br />" % processed
continue
ind["status"] = d.get("Status", Status.NEW)
if not ind["value"] or not ind["type"]:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result["success"] = False
if not ind["value"]:
i += "No valid Indicator value "
if not ind["type"]:
i += "No valid Indicator type "
result_message += "Cannot process row %s: %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, valid_campaigns, "Campaign")
if campaign:
ind["campaign"] = campaign
ind["campaign_confidence"] = get_verified_field(
d, valid_campaign_confidence, "Campaign Confidence", default="low"
)
actions = d.get("Action", "")
if actions:
actions = get_verified_field(actions.split(","), valid_actions)
if not actions:
result["success"] = False
result_message += "Cannot process row %s: Invalid Action<br />" % processed
continue
ind["confidence"] = get_verified_field(d, valid_ratings, "Confidence", default="unknown")
ind["impact"] = get_verified_field(d, valid_ratings, "Impact", default="unknown")
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, "")
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, "")
try:
response = handle_indicator_insert(
ind, source, reference, analyst=username, method=method, add_domain=add_domain
)
except Exception, e:
result["success"] = False
result_message += "Failure processing row %s: %s<br />" % (processed, str(e))
continue
if response["success"]:
if actions:
action = {
"active": "on",
"analyst": username,
"begin_date": "",
"end_date": "",
"performed_date": "",
"reason": "",
"date": datetime.datetime.now(),
}
for action_type in actions:
action["action_type"] = action_type
action_add("Indicator", response.get("objectid"), action, user=username)
else:
result["success"] = False
result_message += "Failure processing row %s: %s<br />" % (processed, response["message"])
continue
added += 1
def handle_indicator_csv(csv_data, source, method, reference, ctype, username,
add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param method: The method of acquisition of this indicator.
:type method: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode('ascii')
data = csv.DictReader(StringIO(cdata), skipinitialspace=True)
result = {'success': True}
result_message = ""
# Compute permitted values in CSV
valid_ratings = {
'unknown': 'unknown',
'benign': 'benign',
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower().replace(' - ', '-')] = c['name']
valid_actions = {}
for a in IndicatorAction.objects(active='on'):
valid_actions[a['name'].lower().replace(' - ', '-')] = a['name']
valid_ind_types = {}
for obj in ObjectType.objects(datatype__enum__exists=False, datatype__file__exists=False):
if obj['object_type'] == obj['name']:
name = obj['object_type']
else:
name = "%s - %s" % (obj['object_type'], obj['name'])
valid_ind_types[name.lower().replace(' - ', '-')] = name
# Start line-by-line import
added = 0
for processed, d in enumerate(data, 1):
ind = {}
ind['value'] = d.get('Indicator', '').lower().strip()
ind['type'] = get_verified_field(d, valid_ind_types, 'Type')
if not ind['value'] or not ind['type']:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result['success'] = False
if not ind['value']:
i += "No valid Indicator value "
if not ind['type']:
i += "No valid Indicator type "
result_message += "Cannot process row %s: %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, valid_campaigns, 'Campaign')
if campaign:
ind['campaign'] = campaign
ind['campaign_confidence'] = get_verified_field(d, valid_campaign_confidence,
'Campaign Confidence',
default='low')
actions = d.get('Action', '')
if actions:
actions = get_verified_field(actions.split(','), valid_actions)
if not actions:
result['success'] = False
result_message += "Cannot process row %s: Invalid Action<br />" % processed
continue
ind['confidence'] = get_verified_field(d, valid_ratings, 'Confidence',
default='unknown')
ind['impact'] = get_verified_field(d, valid_ratings, 'Impact',
default='unknown')
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, '')
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, '')
try:
response = handle_indicator_insert(ind, source, reference, analyst=username,
method=method, add_domain=add_domain)
except Exception, e:
result['success'] = False
result_message += "Failure processing row %s: %s<br />" % (processed, str(e))
continue
if response['success']:
if actions:
action = {'active': 'on',
'analyst': username,
'begin_date': '',
'end_date': '',
'performed_date': '',
'reason': '',
'date': datetime.datetime.now()}
for action_type in actions:
action['action_type'] = action_type
action_add(response.get('objectid'), action)
else:
result['success'] = False
result_message += "Failure processing row %s: %s<br />" % (processed, response['message'])
continue
added += 1
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def handle_indicator_csv(csv_data, source, reference, ctype, username,
add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode('ascii')
data = csv.DictReader(StringIO(cdata), skipinitialspace=True)
result = {'success': True}
result_message = "Indicators added successfully!"
# Compute permitted values in CSV
valid_ratings = {
'unknown': 'unknown',
'benign': 'benign',
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
valid_ind_types = {}
for obj in ObjectType.objects(datatype__enum__exists=False, datatype__file__exists=False):
if obj['object_type'] == obj['name']:
name = obj['object_type']
else:
name = "%s - %s" % (obj['object_type'], obj['name'])
valid_ind_types[name.lower()] = name
# Start line-by-line import
processed = 0
for d in data:
processed += 1
ind = {}
ind['value'] = d.get('Indicator', '').lower().strip()
ind['type'] = get_verified_field(d, 'Type', valid_ind_types)
if not ind['value'] or not ind['type']:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result['success'] = False
if not ind['value']:
i += "No valid Indicator value. "
if not ind['type']:
i += "No valid Indicator type. "
result_message += "Cannot process row: %s. %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, 'Campaign', valid_campaigns)
if campaign:
ind['campaign'] = campaign
ind['campaign_confidence'] = get_verified_field(d, 'Campaign Confidence',
valid_campaign_confidence,
default='low')
ind['confidence'] = get_verified_field(d, 'Confidence', valid_ratings,
default='unknown')
ind['impact'] = get_verified_field(d, 'Impact', valid_ratings,
default='unknown')
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, '')
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, '')
try:
handle_indicator_insert(ind, source, reference, analyst=username,
add_domain=add_domain)
except Exception, e:
result['success'] = False
result['message'] = str(e)
return result
def handle_indicator_csv(csv_data, source, method, reference, ctype, username,
add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param method: The method of acquisition of this indicator.
:type method: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode('ascii')
data = csv.DictReader(StringIO(cdata), skipinitialspace=True)
result = {'success': True}
result_message = ""
# Compute permitted values in CSV
valid_ratings = {
'unknown': 'unknown',
'benign': 'benign',
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower().replace(' - ', '-')] = c['name']
valid_actions = {}
for a in IndicatorAction.objects(active='on'):
valid_actions[a['name'].lower().replace(' - ', '-')] = a['name']
valid_ind_types = {}
for obj in ObjectType.objects(datatype__enum__exists=False, datatype__file__exists=False):
if obj['object_type'] == obj['name']:
name = obj['object_type']
else:
name = "%s - %s" % (obj['object_type'], obj['name'])
valid_ind_types[name.lower().replace(' - ', '-')] = name
# Start line-by-line import
added = 0
for processed, d in enumerate(data, 1):
ind = {}
ind['value'] = d.get('Indicator', '').lower().strip()
ind['type'] = get_verified_field(d, valid_ind_types, 'Type')
if not ind['value'] or not ind['type']:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result['success'] = False
if not ind['value']:
i += "No valid Indicator value "
if not ind['type']:
i += "No valid Indicator type "
result_message += "Cannot process row %s: %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, valid_campaigns, 'Campaign')
if campaign:
ind['campaign'] = campaign
ind['campaign_confidence'] = get_verified_field(d, valid_campaign_confidence,
'Campaign Confidence',
default='low')
actions = d.get('Action', '')
if actions:
actions = get_verified_field(actions.split(','), valid_actions)
if not actions:
result['success'] = False
result_message += "Cannot process row %s: Invalid Action<br />" % processed
continue
ind['confidence'] = get_verified_field(d, valid_ratings, 'Confidence',
default='unknown')
ind['impact'] = get_verified_field(d, valid_ratings, 'Impact',
default='unknown')
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, '')
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, '')
try:
response = handle_indicator_insert(ind, source, reference, analyst=username,
method=method, add_domain=add_domain)
except Exception, e:
result['success'] = False
result_message += "Failure processing row %s: %s<br />" % (processed, str(e))
continue
if response['success']:
if actions:
action = {'active': 'on',
'analyst': username,
'begin_date': '',
'end_date': '',
'performed_date': '',
'reason': '',
'date': datetime.datetime.now()}
for action_type in actions:
action['action_type'] = action_type
action_add(response.get('objectid'), action)
else:
result['success'] = False
result_message += "Failure processing row %s: %s<br />" % (processed, response['message'])
continue
added += 1
def add_new_event(title, description, event_type, source_name, source_method,
source_reference, source_tlp, date, user,
bucket_list=None, ticket=None, campaign=None, campaign_confidence=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param user: The user adding this Event.
:type user: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys "success" (boolean) and "message" (str)
:param campaign: Campaign to associate with this Event
:type campaign: str
:param campaign_confidence: Confidence to associate with the Campaign
:type campaign_confidence: str
"""
if not source_name:
return {'success': False, 'message': "Missing source information."}
result = dict()
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username,
date=date)
else:
return {"success": False,
"message": "User does not have permission to add object \
using source %s." % source_name}
event.add_source(s)
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
if campaign:
if isinstance(campaign, basestring) and len(campaign) > 0:
if campaign.lower() not in valid_campaigns:
result = {'success':False, 'message':'{} is not a valid campaign.'.format(campaign)}
else:
confidence = valid_campaign_confidence.get(campaign_confidence, 'low')
campaign = EmbeddedCampaign(name=campaign,
confidence=confidence,
description="",
analyst=user.username,
date=datetime.datetime.now())
event.add_campaign(campaign)
if bucket_list:
event.add_bucket_list(bucket_list, user.username)
if ticket:
event.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
event.save(username=user.username)
if related_obj and event and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
event.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
event.save(username=user.username)
# run event triage
event.reload()
run_triage(event, user.username)
message = ('<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' % (reverse('crits.events.views.view_event',
args=[event.id]),
title))
result = {'success': True,
'message': message,
'id': str(event.id),
'object': event}
except ValidationError, e:
result = {'success': False,
'message': e}
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatType, ActorMotivation
from crits.actors.actor import ActorSophistication, ActorIntendedEffect
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Comment', 'Email', 'Event', 'Indicator', 'Screenshot']
and not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'ActorThreatType':
return ActorThreatType.objects(name=value).first()
elif type_ == 'ActorMotivation':
return ActorMotivation.objects(name=value).first()
elif type_ == 'ActorSophistication':
return ActorSophistication.objects(name=value).first()
elif type_ == 'ActorIntendedEffect':
return ActorIntendedEffect.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
'Indicator', 'Screenshot'] and
not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def add_campaign(name, description, aliases, analyst,
bucket_list=None, ticket=None, related_id=None,
related_type=None, relationship_type=None):
"""
Add a Campaign.
:param name: The name of the new Campaign.
:type name: str
:param description: Description of the new Campaign.
:type description: str
:param aliases: Aliases for the new Campaign.
:type aliases: str (comma separated) or list.
:param analyst: The user adding the Campaign.
:type analyst: str
:param bucket_list: Buckets to add to this Campaign.
:type bucket_list: str (comma separated) or list.
:param ticket: Ticket(s) to add to this Campaign.
:type ticket: str (comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with key 'success' (boolean) and 'message' (str).
"""
# Verify the Campaign does not exist.
campaign = Campaign.objects(name=name).first()
if campaign:
return {'success': False, 'message': ['Campaign already exists.'],
'id': str(campaign.id)}
# Create new campaign.
campaign = Campaign(name=name)
campaign.edit_description(description)
if bucket_list:
campaign.add_bucket_list(bucket_list, analyst)
if ticket:
campaign.add_ticket(ticket, analyst)
# Adjust aliases.
if isinstance(aliases, basestring):
alias_list = aliases.split(',')
final_aliases = [a.strip() for a in alias_list]
elif isinstance(aliases, list):
final_aliases = [a.strip() for a in aliases]
else:
final_aliases = []
campaign.add_alias(final_aliases)
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
campaign.save(username=analyst)
if related_obj and relationship_type and campaign:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
campaign.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
campaign.save(username=analyst)
campaign.reload()
try:
campaign.save(username=analyst)
campaign.reload()
return {'success': True,
'message': 'Campaign created successfully!',
'id': str(campaign.id)}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def add_campaign(name,
description,
aliases,
analyst,
bucket_list=None,
ticket=None):
"""
Add a Campaign.
:param name: The name of the new Campaign.
:type name: str
:param description: Description of the new Campaign.
:type description: str
:param aliases: Aliases for the new Campaign.
:type aliases: str (comma separated) or list.
:param analyst: The user adding the Campaign.
:type analyst: str
:param bucket_list: Buckets to add to this Campaign.
:type bucket_list: str (comma separated) or list.
:param ticket: Ticket(s) to add to this Campaign.
:type ticket: str (comma separated) or list.
:returns: dict with key 'success' (boolean) and 'message' (str).
"""
# Verify the Campaign does not exist.
campaign = Campaign.objects(name=name).first()
if campaign:
return {
'success': False,
'message': ['Campaign already exists.'],
'id': str(campaign.id)
}
# Create new campaign.
campaign = Campaign(name=name)
campaign.edit_description(description)
if bucket_list:
campaign.add_bucket_list(bucket_list, analyst)
if ticket:
campaign.add_ticket(ticket, analyst)
# Adjust aliases.
if isinstance(aliases, basestring):
alias_list = aliases.split(',')
final_aliases = [a.strip() for a in alias_list]
elif isinstance(aliases, list):
final_aliases = [a.strip() for a in aliases]
else:
final_aliases = []
campaign.add_alias(final_aliases)
try:
campaign.save(username=analyst)
campaign.reload()
return {
'success': True,
'message': 'Campaign created successfully!',
'id': str(campaign.id)
}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def add_new_event(title, description, event_type, source_name, source_method,
source_reference, source_tlp, date, user,
bucket_list=None, ticket=None, campaign=None, campaign_confidence=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param user: The user adding this Event.
:type user: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys "success" (boolean) and "message" (str)
:param campaign: Campaign to associate with this Event
:type campaign: str
:param campaign_confidence: Confidence to associate with the Campaign
:type campaign_confidence: str
"""
if not source_name:
return {'success': False, 'message': "Missing source information."}
result = dict()
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username,
date=date)
else:
return {"success": False,
"message": "User does not have permission to add object \
using source %s." % source_name}
event.add_source(s)
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
if campaign:
if isinstance(campaign, basestring) and len(campaign) > 0:
if campaign.lower() not in valid_campaigns:
result = {'success':False, 'message':'{} is not a valid campaign.'.format(campaign)}
else:
confidence = valid_campaign_confidence.get(campaign_confidence, 'low')
campaign = EmbeddedCampaign(name=campaign,
confidence=confidence,
description="",
analyst=user.username,
date=datetime.datetime.now())
event.add_campaign(campaign)
if bucket_list:
event.add_bucket_list(bucket_list, user.username)
if ticket:
event.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
event.save(username=user.username)
if related_obj and event and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
event.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
event.save(username=user.username)
# run event triage
event.reload()
run_triage(event, user.username)
message = ('<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' % (reverse('crits-events-views-view_event',
args=[event.id]),
title))
result = {'success': True,
'message': message,
'id': str(event.id),
'object': event}
except ValidationError, e:
result = {'success': False,
'message': e}
def get_campaign_details(campaign_name, analyst):
"""
Generate the data to render the Campaign details template.
:param campaign_name: The name of the Campaign to get details for.
:type campaign_name: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
campaign_detail = Campaign.objects(name=campaign_name).first()
if not campaign_detail:
template = "error.html"
args = {"error": 'No data exists for this campaign.'}
return template, args
campaign_detail.sanitize(username=analyst)
ttp_form = TTPForm()
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, campaign_detail.id, 'Campaign')
# subscription
subscription = {
'type': 'Campaign',
'id': campaign_detail.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Campaign',
campaign_detail.id),
}
#objects
objects = campaign_detail.sort_objects()
#relationships
relationships = campaign_detail.sort_relationships("%s" % analyst,
meta=True)
# relationship
relationship = {'type': 'Campaign', 'value': campaign_detail.id}
#comments
comments = {'comments': campaign_detail.get_comments(),
'url_key': campaign_name}
#screenshots
screenshots = campaign_detail.get_screenshots(analyst)
# Get item counts
formatted_query = {'campaign.name': campaign_name}
counts = {}
for col_obj in [Actor, Backdoor, Exploit, Sample, PCAP, Indicator, Email, Domain, IP, Event]:
counts[col_obj._meta['crits_type']] = col_obj.objects(source__name__in=sources,
__raw__=formatted_query).count()
# Item counts for targets
uniq_addrs = get_campaign_targets(campaign_name, analyst)
counts['Target'] = Target.objects(email_address__in=uniq_addrs).count()
# favorites
favorite = is_user_favorite("%s" % analyst, 'Campaign', campaign_detail.id)
# analysis results
service_results = campaign_detail.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
"relationship": relationship,
'comments': comments,
"subscription": subscription,
"campaign_detail": campaign_detail,
"counts": counts,
"favorite": favorite,
"screenshots": screenshots,
'service_results': service_results,
"ttp_form": ttp_form,
"CampaignACL": CampaignACL}
return template, args
def campaign_heatmap(request):
campaigns = Campaign.objects().only('name', 'aliases', 'locations')
events = Event.objects().only('title', 'created', 'locations', 'campaign')
emails = Email.objects().only('created', 'locations', 'campaign')
# list of countries in alphabetical order. set 0 for the amount of campaign
# associated with this country for the next step.
country_list = []
for c in campaigns:
if len(c.locations):
for l in c.locations:
if [l.location,0] not in country_list:
country_list.append([l.location,0])
country_list.sort()
# For those campaigns with no location assigned, have an Unknown location.
country_list.append(['Unknown', 0])
# list of campaigns in order of country, then alphabetical by name
campaign_list = []
# for each country we build a tmp list, find all campaigns for that country,
# sort the list, then append it to the campaign list. bump the count so we
# know how many columns to span.
for c in country_list:
tmp = []
for cam in campaigns:
if len(cam.locations):
for l in cam.locations:
if l.location == c[0]:
c[1] += 1
if cam.name not in tmp:
tmp.append(cam.name)
break
else:
# Assuming we are checking the Unknown location, if this
# campaign has no location assigned, add it to Unknown.
if c[0] == 'Unknown':
c[1] += 1
if cam.name not in tmp:
tmp.append(cam.name)
# If we haven't added a campaign to this location, show "No Campaigns".
# This also prevents a left-shift in the counting and header rows.
if len(tmp) == 0:
tmp.append("No Campaigns")
tmp.sort()
campaign_list += tmp
# list of the months going back in history and the activity of each campaign
# during that month
month_list = []
# for each campaign, find associated events and emails. For each event and
# email, use the created date to put it into the appropriate list.
month_d = {}
idx = 0
# this is a default row in the heatmap with all values set to 0.
pad_list = [0 for _ in range(len(campaign_list))]
for c in campaign_list:
build_month_d(pad_list, month_d, c, idx, events)
build_month_d(pad_list, month_d, c, idx, emails)
idx += 1
# sort the months in reverse order for descending display.
for key in sorted(month_d, reverse=True):
month_list.append([key, month_d[key]])
final_data = {
'country_list': country_list,
'campaign_list': campaign_list,
'month_list': month_list,
}
return final_data
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# Quick fail
if not _id or not type_:
return None
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode("utf8")):
return None
if type_ == "Actor":
return Actor.objects(id=_id).first()
elif type_ == "Backdoor":
return Backdoor.objects(id=_id).first()
elif type_ == "ActorThreatIdentifier":
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == "Campaign":
return Campaign.objects(id=_id).first()
elif type_ == "Certificate":
return Certificate.objects(id=_id).first()
elif type_ == "Comment":
return Comment.objects(id=_id).first()
elif type_ == "Domain":
return Domain.objects(id=_id).first()
elif type_ == "Email":
return Email.objects(id=_id).first()
elif type_ == "Event":
return Event.objects(id=_id).first()
elif type_ == "Exploit":
return Exploit.objects(id=_id).first()
elif type_ == "Indicator":
return Indicator.objects(id=_id).first()
elif type_ == "Action":
return Action.objects(id=_id).first()
elif type_ == "IP":
return IP.objects(id=_id).first()
elif type_ == "PCAP":
return PCAP.objects(id=_id).first()
elif type_ == "RawData":
return RawData.objects(id=_id).first()
elif type_ == "RawDataType":
return RawDataType.objects(id=_id).first()
elif type_ == "Sample":
return Sample.objects(id=_id).first()
elif type_ == "Signature":
return Signature.objects(id=_id).first()
elif type_ == "SignatureType":
return SignatureType.objects(id=_id).first()
elif type_ == "SignatureDependency":
return SignatureDependency.objects(id=_id).first()
elif type_ == "SourceAccess":
return SourceAccess.objects(id=_id).first()
elif type_ == "Screenshot":
return Screenshot.objects(id=_id).first()
elif type_ == "Target":
return Target.objects(id=_id).first()
elif type_ == "UserRole":
return UserRole.objects(id=_id).first()
else:
return None