def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if 'email_address' not in data:
return {'success': False,
'message': "No email address to look up"}
target = Target.objects(email_address__iexact=data['email_address']).first()
is_new = False
if not target:
is_new = True
target = Target()
target.email_address = data['email_address']
bucket_list = False
ticket = False
if 'department' in data:
target.department = data['department']
if 'division' in data:
target.division = data['division']
if 'organization_id' in data:
target.organization_id = data['organization_id']
if 'firstname' in data:
target.firstname = data['firstname']
if 'lastname' in data:
target.lastname = data['lastname']
if 'note' in data:
target.note = data['note']
if 'title' in data:
target.title = data['title']
if 'bucket_list' in data:
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if 'ticket' in data:
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
try:
target.save(username=analyst)
target.reload()
if is_new:
run_triage(target, analyst)
return {'success': True,
'message': "Target saved successfully",
'id': str(target.id)}
except ValidationError, e:
return {'success': False,
'message': "Target save failed: %s" % e}
def run_analysis_cleanup(self, obj_list, type_, delay):
print "\nCleaning Analysis for:\n---------------"
for obj in obj_list:
results = AnalysisResult.objects(object_type=type_, object_id=str(obj.id))
print(" [+] {0}".format(obj.id))
for result in results:
result.delete()
time.sleep(float(delay))
run_triage(obj, self.username)
def handle_indicator_insert(ind, source, reference='', analyst='', method='',
add_domain=False, add_relationship=False, cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]:
return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "}
is_new_indicator = False
dmain = None
ip = None
rank = {
'unknown': 0,
'benign': 1,
'low': 2,
'medium': 3,
'high': 4,
}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
if 'campaign' in ind:
if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0:
confidence = ind.get('campaign_confidence', 'low')
ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if isinstance(ind['campaign'], EmbeddedCampaign):
indicator.add_campaign(ind['campaign'])
elif isinstance(ind['campaign'], list):
for campaign in ind['campaign']:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
url_contains_ip = False
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain_or_ip = urlparse.urlparse(ind_value).hostname
elif ind_type == "URI - Domain Name":
domain_or_ip = ind_value
(sdomain, fqdn) = get_domain(domain_or_ip)
if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(sdomain, fqdn, indicator.source,
'%s' % analyst, None,
bucket_list=bucket_list, cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success['object']
if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = 'Address - ipv4-addr'
except DjangoValidationError:
ind_type = 'Address - ipv6-addr'
success = None
if add_domain:
success = ip_add_update(ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {'success': True, 'objectid': str(indicator.id),
'is_new_indicator': is_new_indicator, 'object': indicator}
def add_new_backdoor(name, version=None, aliases=None, description=None,
source=None, source_method=None, source_reference=None,
campaign=None, confidence=None, user=None,
bucket_list=None, ticket=None):
"""
Add an Backdoor to CRITs.
:param name: The name of the backdoor.
:type name: str
:param version: Version of the backdoor.
:type version: str
:param aliases: Aliases for the backdoor.
:type aliases: list or str
:param description: Description of the backdoor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this backdoor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this backdoor.
:type user: str
:param bucket_list: Buckets to assign to this backdoor.
:type bucket_list: str
:param ticket: Ticket to assign to this backdoor.
:type ticket: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"id" (str),
"object" (if successful) :class:`crits.backdoors.backdoor.Backdoor`
"""
retVal = {'success': False, 'message': ''}
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=user)]
elif isinstance(source, EmbeddedSource):
source = [source]
if not source:
retVal['message'] = "Missing source information."
return retVal
# When creating a backdoor object we can potentially create multiple
# objects. If we are given a name but no version we will create an object
# with just the name (called the "family backdoor"). If given a name and a
# version we will create the family backdoor and the specific backdoor for
# that given version.
# In case we create more than one backdoor object, store the created ones
# in this list. The list is walked later on and attributes applied to each
# object.
objs = []
# First check if we have the family (name and no version).
family = Backdoor.objects(name=name, version='').first()
if not family:
# Family does not exist, new object. Details are handled later.
family = Backdoor()
family.name = name
family.version = ''
objs.append(family)
# Now check if we have the specific instance for this name + version.
backdoor = None
if version:
backdoor = Backdoor.objects(name=name, version=version).first()
if not backdoor:
# Backdoor does not exist, new object. Details are handled later.
backdoor = Backdoor()
backdoor.name = name
backdoor.version = version
objs.append(backdoor)
# At this point we have a family object and potentially a specific object.
# Add the common parameters to all objects in the list and save them.
for backdoor in objs:
for s in source:
backdoor.add_source(s)
# Don't overwrite existing description.
if description and backdoor.description == '':
backdoor.description = description.strip()
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=user)
campaign = [c]
if campaign:
for camp in campaign:
backdoor.add_campaign(camp)
if aliases:
if isinstance(aliases, basestring):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in backdoor.aliases:
backdoor.aliases.append(alias)
if bucket_list:
backdoor.add_bucket_list(bucket_list, user)
if ticket:
backdoor.add_ticket(ticket, user)
backdoor.save(username=user)
# run backdoor triage
backdoor.reload()
run_triage(backdoor, user)
# Because family objects are put in the list first we will always
# return a link to the most specific object created. If there is only
# one item in the list it will be the family object.
resp_url = reverse('crits.backdoors.views.backdoor_detail',
args=[backdoor.id])
retVal['message'] = 'Success: <a href="%s">%s</a>' % (resp_url,
backdoor.name)
retVal['object'] = backdoor
retVal['id'] = str(backdoor.id)
# If we have a family and specific object, attempt to relate the two.
if len(objs) == 2:
objs[0].add_relationship(objs[1], RelationshipTypes.RELATED_TO)
objs[0].save()
retVal['success'] = True
return retVal
def add_new_event(title, description, event_type, source_name, source_method,
source_reference, source_tlp, date, user,
bucket_list=None, ticket=None, campaign=None, campaign_confidence=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param user: The user adding this Event.
:type user: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys "success" (boolean) and "message" (str)
:param campaign: Campaign to associate with this Event
:type campaign: str
:param campaign_confidence: Confidence to associate with the Campaign
:type campaign_confidence: str
"""
if not source_name:
return {'success': False, 'message': "Missing source information."}
result = dict()
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username,
date=date)
else:
return {"success": False,
"message": "User does not have permission to add object \
using source %s." % source_name}
event.add_source(s)
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
if campaign:
if isinstance(campaign, basestring) and len(campaign) > 0:
if campaign.lower() not in valid_campaigns:
result = {'success':False, 'message':'{} is not a valid campaign.'.format(campaign)}
else:
confidence = valid_campaign_confidence.get(campaign_confidence, 'low')
campaign = EmbeddedCampaign(name=campaign,
confidence=confidence,
description="",
analyst=user.username,
date=datetime.datetime.now())
event.add_campaign(campaign)
if bucket_list:
event.add_bucket_list(bucket_list, user.username)
if ticket:
event.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
event.save(username=user.username)
if related_obj and event and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
event.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
event.save(username=user.username)
# run event triage
event.reload()
run_triage(event, user.username)
message = ('<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' % (reverse('crits-events-views-view_event',
args=[event.id]),
title))
result = {'success': True,
'message': message,
'id': str(event.id),
'object': event}
except ValidationError, e:
result = {'success': False,
'message': e}
def add_new_exploit(name, cve=None, description=None, source=None,
source_method=None, source_reference=None, source_tlp=None,
campaign=None, confidence=None, user=None,
bucket_list=None, ticket=None,related_id=None,
related_type=None, relationship_type=None):
"""
Add an Exploit to CRITs.
:param name: The name of the exploit.
:type name: str
:param cve: CVE for the exploit.
:type cve: str
:param description: Description of the exploit.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param source_tlp: TLP value for this source
:type source_reference: str
:param campaign: A campaign to attribute to this exploit.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this exploit.
:type user: str
:param bucket_list: Buckets to assign to this exploit.
:type bucket_list: str
:param ticket: Ticket to assign to this exploit.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.exploits.exploit.Exploit`
"""
is_item_new = False
retVal = {}
exploit = Exploit.objects(name=name).first()
if not exploit:
exploit = Exploit()
exploit.name = name
if description:
exploit.description = description.strip()
is_item_new = True
username = user.username
if isinstance(source, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=username)]
else:
return {"success": False,
"message": "User does not have permission to add object \
using source %s." % source}
elif isinstance(source, EmbeddedSource):
source = [source]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
campaign = [c]
if campaign:
for camp in campaign:
exploit.add_campaign(camp)
if source:
for s in source:
exploit.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
exploit.cve = cve.strip()
if bucket_list:
exploit.add_bucket_list(bucket_list, user)
if ticket:
exploit.add_ticket(ticket, user)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
exploit.save(username=username)
if related_obj and exploit and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
exploit.add_relationship(related_obj,
relationship_type,
analyst=user,
get_rels=False)
exploit.save(username=username)
# run exploit triage
if is_item_new:
exploit.reload()
run_triage(exploit, username)
resp_url = reverse('crits-exploits-views-exploit_detail', args=[exploit.id])
retVal['message'] = ('Success! Click here to view the new Exploit: '
'<a href="%s">%s</a>' % (resp_url, exploit.name))
retVal['success'] = True
retVal['object'] = exploit
retVal['id'] = str(exploit.id)
return retVal
def handle_cert_file(filename,
data,
source_name,
user=None,
description=None,
parent_id=None,
parent_md5=None,
parent_type=None,
method=None,
relationship=None,
bucket_list=None,
ticket=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param parent_id: ObjectId of the top-level object where this Certificate
came from.
:type parent_id: str
:param parent_md5: MD5 of the top-level object where this Certificate
came from.
:type parent_md5: str
:param parent_type: The CRITs type of the parent.
:type parent_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference='',
analyst=user)
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if len(cert.analysis) < 1 and data:
run_triage(data, cert, user)
# update parent relationship if a parent is supplied
if parent_id or parent_md5:
if not relationship:
relationship = "Related_To"
if parent_id:
parent_obj = class_from_id(parent_type, parent_id)
else:
parent_obj = class_from_value(parent_type, parent_md5)
if parent_obj and cert:
cert.add_relationship(rel_item=parent_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
parent_obj.save(username=user)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
}
return status
def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method='', reference='', tlp='',
copy_rels=False, bucket_list=None, ticket=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param reference: A reference to the source of this RawData.
:type reference: str
:param tlp: TLP for the source.
:type tlp: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if isinstance(data, unicode):
data=data.encode('utf-8')
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if not raw_data:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
# raw_data.source = [source_name]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
source = create_embedded_source(source_name,
method=method,
reference=reference,
tlp=tlp,
analyst=user.username)
raw_data.add_source(source)
else:
return {"success":False,
"message": "User does not have permission to add object using source %s." % source_name}
# this will handle adding a new source, or an instance automatically
elif isinstance(source_name, EmbeddedSource):
raw_data.add_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.usrname)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
raw_data.add_source(s, method=method, reference=reference, tlp=tlp, analyst=user.username)
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
# Get object to relate to.
rel_item = class_from_id(rel.rel_type, rel.object_id)
if rel_item:
raw_data.add_relationship(rel_item,
rel.relationship,
rel_date=rel.relationship_date,
analyst=user.username)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
raw_data.save(username=user.username)
if related_obj and relationship_type and raw_data:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
raw_data.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
raw_data.save(username=user.username)
raw_data.reload()
# save raw_data
raw_data.save(username=user.username)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
'object': raw_data
}
return status
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(None, indicator, analyst)
return {'success': True, 'objectid': indicator.id,
'is_new_indicator': is_new_indicator, 'object': indicator}
def does_indicator_relationship_exist(field, indicator_relationships):
"""
Checks if the input field's values already have an indicator
by cross checking against the list of indicator relationships. The input
field already has an associated indicator created if the input field's
"type" and "value" pairs exist -- since indicators are uniquely identified
by their type/value pair.
Args:
field: The generic input field containing a type/value pair. This is
checked against a list of indicators relationships to see if a
def run(self, obj, config):
key = config.get('vt_api_key', '')
url = config.get('vt_download_url', '')
sizeLimit = config.get('size_limit', '')
replace = config.get('replace_sample', False)
do_triage = config.get('run_triage', False)
user = self.current_task.user
sample = Sample.objects(md5=obj.md5).first()
if not sample:
sample = Sample()
sample.md5 = md5_digest
self._info("Checking if binary already exists in CRITs.")
sample.discover_binary()
if sample.filedata and replace == False: #if we already have this binary and don't have permission to replace
self._info(
"CRITs already has this binary. Enable the 'Replace' option to overwrite with data from VirusTotal."
)
self._add_result("Download Canceled",
"Binary already exists in CRITs.")
return
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result(
"Download Canceled",
"User does not have permission to add Samples to CRITs")
return
parameters = urllib.urlencode({"hash": obj.md5, "apikey": key})
if settings.HTTP_PROXY:
proxy = urllib2.ProxyHandler({
'http': settings.HTTP_PROXY,
'https': settings.HTTP_PROXY
})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
try:
req = url + "?" + parameters
self._info(
"Requesting binary with md5 '{0}' from VirusTotal.".format(
obj.md5))
request = urllib2.Request(req)
response = urllib2.urlopen(request)
size = response.info().getheaders("Content-Length")[0]
self._info("Binary size: {0} bytes".format(size))
if int(size) > sizeLimit: # Check if within size limit
self._error(
"Binary size is {0} bytes, which is greater than maximum of {1} bytes. This limit can be changed in options."
.format(size, sizeLimit))
self._add_result(
"Download Aborted",
"Match found, but binary is larger than maximum size limit."
)
return
data = response.read()
except urllib2.HTTPError as e:
if e.code == 404:
self._info(
"No results were returned. Either VirusTotal does not have the requested binary, or the request URL is incorrect."
)
self._add_result(
"Not Found",
"Binary was not found in the VirusTotal database")
elif e.code == 403:
self._error("Download forbidden. {0}".format(e))
self._add_result(
"Download Canceled",
"CRITs was forbidden from downloading the binary.")
else:
self._error("An HTTP Error occurred: {0}".format(e))
return
except Exception as e:
logger.error("VirusTotal: Failed connection ({0})".format(e))
self._error("Failed to get data from VirusTotal: {0}".format(e))
return
if data: # Retrieved some data from VT
if replace == True:
try:
self._info(
"Replace = True. Deleting any previous binary with md5 {0}"
.format(obj.md5))
sample.filedata.delete()
except Exception as e:
logger.error(
"VirusTotal: Error deleting existing binary ({0})".
format(e))
self._error("Failed to delete existing binary")
self._info("Adding new binary to CRITs.")
try:
handle_file(filename=obj.md5,
data=data,
source="VirusTotal",
reference="Binary downloaded from VT based on MD5",
user="******",
method="VirusTotal Download Service",
md5_digest=obj.md5)
except Exception as e:
logger.error(
"VirusTotal: Sample creation failed ({0})".format(e))
self._error("Failed to create new Sample: {0}".format(e))
return
if do_triage:
self._info("Running sample triage for data-reliant services.")
sample.reload()
run_triage(sample, user="******")
self._add_result(
"Download Successful",
"Binary was successfully downloaded from VirusTotal")
else:
self._error("No data returned by VirusTotal.")
def handle_cert_file(filename,
data,
source_name,
user=None,
description=None,
related_md5=None,
method='',
reference='',
tlp=None,
relationship=None,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param reference: A reference to the source of this Certificate.
:type reference: str
:param tlp: The TLP for this certificate.
:type tlp: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
if ((related_type and not (related_id or related_md5))
or (not related_type and (related_id or related_md5))):
status = {
'success':
False,
'message':
'Must specify both related_type and related_id or related_md5.'
}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {'success': False, 'message': 'Related object not found.'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
reference=reference,
method=method,
tlp=tlp,
analyst=user.username)
else:
return {
"success":
False,
"message":
"User does not have permission to add objects \
using source %s." % str(source_name)
}
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name,
method=method,
reference=reference,
tlp=tlp)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s, method=method, reference=reference, tlp=tlp)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if len(AnalysisResult.objects(object_id=str(cert.id))) < 1 and data:
run_triage(cert, user)
# update relationship if a related top-level object is supplied
if related_obj and cert:
if relationship_type:
relationship = RelationshipTypes.inverse(
relationship=relationship_type)
if not relationship:
relationship = RelationshipTypes.RELATED_TO
cert.add_relationship(related_obj,
relationship,
analyst=user,
get_rels=False)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
'id': str(cert.id),
'object': cert
}
return status
def add_new_event(title,
description,
event_type,
source,
method,
reference,
date,
analyst,
bucket_list=None,
ticket=None,
campaign=None,
campaign_confidence=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param analyst: The user adding this Event.
:type analyst: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys "success" (boolean) and "message" (str)
:param campaign: Campaign to associate with this Event
:type campaign: str
:param campaign_confidence: Confidence to associate with the Campaign
:type campaign_confidence: str
"""
result = dict()
if not source:
return {'success': False, 'message': "Missing source information."}
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
s = create_embedded_source(name=source,
reference=reference,
method=method,
analyst=analyst,
date=date)
event.add_source(s)
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'
}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
if campaign:
if isinstance(campaign, basestring) and len(campaign) > 0:
if campaign.lower() not in valid_campaigns:
result = {
'success': False,
'message': '{} is not a valid campaign.'.format(campaign)
}
else:
confidence = valid_campaign_confidence.get(
campaign_confidence, 'low')
campaign = EmbeddedCampaign(name=campaign,
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
event.add_campaign(campaign)
if bucket_list:
event.add_bucket_list(bucket_list, analyst)
if ticket:
event.add_ticket(ticket, analyst)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
event.save(username=analyst)
if related_obj and event and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
event.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
event.save(username=analyst)
# run event triage
event.reload()
run_triage(event, analyst)
message = (
'<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' %
(reverse('crits.events.views.view_event', args=[event.id]), title))
result = {
'success': True,
'message': message,
'id': str(event.id),
'object': event
}
except ValidationError, e:
result = {'success': False, 'message': e}
def handle_indicator_insert(ind,
source,
reference='',
analyst='',
method='',
add_domain=False,
add_relationship=False,
cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
is_new_indicator = False
rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
if 'campaign' in ind:
if isinstance(ind['campaign'],
basestring) and len(ind['campaign']) > 0:
confidence = ind.get('campaign_confidence', 'low')
ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if isinstance(ind['campaign'], EmbeddedCampaign):
indicator.add_campaign(ind['campaign'])
elif isinstance(ind['campaign'], list):
for campaign in ind['campaign']:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(
indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(
indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
indicator.save(username=analyst)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
url_contains_ip = False
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain_or_ip = urlparse.urlparse(ind_value).hostname
elif ind_type == "URI - Domain Name":
domain_or_ip = ind_value
#try:
(sdomain, fqdn) = get_domain(domain_or_ip)
if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(sdomain,
fqdn,
indicator.source,
'%s' % analyst,
None,
bucket_list=bucket_list,
cache=cache)
if not success['success']:
indicator_link = '<a href=\"%s\">View indicator</a>.</div>' \
% (reverse('crits.indicators.views.indicator', args=[indicator.id]))
message = "Indicator was added, but an error occurred. " + indicator_link + "<br>"
return {
'success': False,
'message': message + success['message'],
'id': str(indicator.id)
}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success['object']
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
indicator.save(username=analyst)
if ind_type.startswith(
"Address - ip"
) or ind_type == "Address - cidr" or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
success = None
if add_domain:
success = ip_add_update(ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
indicator_link = '<a href=\"%s\">View indicator</a>.</div>' \
% (reverse('crits.indicators.views.indicator', args=[indicator.id]))
message = "Indicator was added, but an error occurred. " + indicator_link + "<br>"
return {
'success': False,
'message': message + success['message'],
'id': str(indicator.id)
}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {
'success': True,
'objectid': str(indicator.id),
'is_new_indicator': is_new_indicator,
'object': indicator
}
def run(self, obj, config):
key = config.get('vt_api_key', '')
url = config.get('vt_download_url', '')
sizeLimit = config.get('size_limit', '')
replace = config.get('replace_sample', False)
do_triage = config.get('run_triage', False)
user = self.current_task.user
sample = Sample.objects(md5=obj.md5).first()
if not sample:
sample = Sample()
sample.md5 = md5_digest
self._info("Checking if binary already exists in CRITs.")
sample.discover_binary()
if sample.filedata and replace == False: #if we already have this binary and don't have permission to replace
self._info("CRITs already has this binary. Enable the 'Replace' option to overwrite with data from VirusTotal.")
self._add_result("Download Canceled", "Binary already exists in CRITs.")
return
if not user.has_access_to(SampleACL.WRITE):
self._info("User does not have permission to add Samples to CRITs")
self._add_result("Download Canceled", "User does not have permission to add Samples to CRITs")
return
parameters = urllib.urlencode({"hash": obj.md5, "apikey": key})
if settings.HTTP_PROXY:
proxy = urllib2.ProxyHandler({'http': settings.HTTP_PROXY, 'https': settings.HTTP_PROXY})
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
try:
req = url + "?" + parameters
self._info("Requesting binary with md5 '{0}' from VirusTotal.".format(obj.md5))
request = urllib2.Request(req)
response = urllib2.urlopen(request)
size = response.info().getheaders("Content-Length")[0]
self._info("Binary size: {0} bytes".format(size))
if int(size) > sizeLimit: # Check if within size limit
self._error("Binary size is {0} bytes, which is greater than maximum of {1} bytes. This limit can be changed in options.".format(size, sizeLimit))
self._add_result("Download Aborted", "Match found, but binary is larger than maximum size limit.")
return
data = response.read()
except urllib2.HTTPError as e:
if e.code == 404:
self._info("No results were returned. Either VirusTotal does not have the requested binary, or the request URL is incorrect.")
self._add_result("Not Found", "Binary was not found in the VirusTotal database")
elif e.code == 403:
self._error("Download forbidden. {0}".format(e))
self._add_result("Download Canceled", "CRITs was forbidden from downloading the binary.")
else:
self._error("An HTTP Error occurred: {0}".format(e))
except Exception as e:
logger.error("VirusTotal: Failed connection ({0})".format(e))
self._error("Failed to get data from VirusTotal: {0}".format(e))
return
if data: # Retrieved some data from VT
if replace == True:
try:
self._info("Replace = True. Deleting any previous binary with md5 {0}".format(obj.md5))
sample.filedata.delete()
except Exception as e:
logger.error("VirusTotal: Error deleting existing binary ({0})".format(e))
self._error("Failed to delete existing binary")
self._info("Adding new binary to CRITs.")
try:
handle_file(filename = obj.md5,
data = data,
source = "VirusTotal",
reference = "Binary downloaded from VT based on MD5",
user = "******",
method = "VirusTotal Download Service",
md5_digest = obj.md5 )
except Exception as e:
logger.error("VirusTotal: Sample creation failed ({0})".format(e))
self._error("Failed to create new Sample: {0}".format(e))
return
if do_triage:
self._info("Running sample triage for data-reliant services.")
sample.reload()
run_triage(sample, user = "******")
self._add_result("Download Successful", "Binary was successfully downloaded from VirusTotal")
else:
self._error("No data returned by VirusTotal.")
def add_new_actor(name, aliases=None, description=None, source=None,
source_method='', source_reference='', campaign=None,
confidence=None, analyst=None, bucket_list=None, ticket=None):
"""
Add an Actor to CRITs.
:param name: The name of the Actor.
:type name: str
:param aliases: Aliases for the actor.
:type aliases: list or str
:param description: Description of the actor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this actor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding this actor.
:type analyst: str
:param bucket_list: Buckets to assign to this actor.
:type bucket_list: str
:param ticket: Ticket to assign to this actor.
:type ticket: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.actors.actor.Actor`
"""
is_item_new = False
retVal = {}
actor = Actor.objects(name=name).first()
if not actor:
actor = Actor()
actor.name = name
if description:
actor.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
actor.add_campaign(camp)
if source:
for s in source:
actor.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if not isinstance(aliases, list):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in actor.aliases:
actor.aliases.append(alias)
if bucket_list:
actor.add_bucket_list(bucket_list, analyst)
if ticket:
actor.add_ticket(ticket, analyst)
actor.save(username=analyst)
# run actor triage
if is_item_new:
actor.reload()
run_triage(actor, analyst)
resp_url = reverse('crits.actors.views.actor_detail', args=[actor.id])
retVal['message'] = ('Success! Click here to view the new Actor: '
'<a href="%s">%s</a>' % (resp_url, actor.name))
retVal['success'] = True
retVal['object'] = actor
retVal['id'] = str(actor.id)
return retVal
def add_new_actor(name, aliases=None, description=None, source=None,
source_method='', source_reference='', source_tlp=None,
campaign=None, confidence=None, user=None,
bucket_list=None, ticket=None, related_id=None,
related_type=None, relationship_type=None):
"""
Add an Actor to CRITs.
:param name: The name of the Actor.
:type name: str
:param aliases: Aliases for the actor.
:type aliases: list or str
:param description: Description of the actor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param source_tlp: The TLP for this Actor.
:type source_tlp: str
:param campaign: A campaign to attribute to this actor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this actor.
:type user: :class:`crits.core.user.CRITsUser`
:param bucket_list: Buckets to assign to this actor.
:type bucket_list: str
:param ticket: Ticket to assign to this actor.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.actors.actor.Actor`
"""
username = user.username
is_item_new = False
retVal = {}
actor = Actor.objects(name=name).first()
if not actor:
actor = Actor()
actor.name = name
if description:
actor.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=username)]
else:
return {"success": False,
"message": "User does not have permission to add objects \
using source %s." % str(source)}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
if campaign:
for camp in campaign:
actor.add_campaign(camp)
if source:
for s in source:
actor.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if not isinstance(aliases, list):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in actor.aliases:
actor.aliases.append(alias)
if bucket_list:
actor.add_bucket_list(bucket_list, username)
if ticket:
actor.add_ticket(ticket, username)
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
actor.save(username=username)
if related_obj and actor:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
actor.add_relationship(related_obj,
relationship_type,
analyst=username,
get_rels=False)
actor.save(username=username)
actor.reload()
# run actor triage
if is_item_new:
actor.reload()
run_triage(actor, user)
resp_url = reverse('crits.actors.views.actor_detail', args=[actor.id])
retVal['message'] = ('Success! Click here to view the new Actor: '
'<a href="%s">%s</a>' % (resp_url, actor.name))
retVal['success'] = True
retVal['object'] = actor
retVal['id'] = str(actor.id)
return retVal
def add_new_event(title, description, event_type, source, method, reference,
date, analyst, bucket_list=None, ticket=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param analyst: The user adding this Event.
:type analyst: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if not source:
return {'success': False, 'message': "Missing source information."}
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
s = create_embedded_source(name=source,
reference=reference,
method=method,
analyst=analyst,
date=date)
event.add_source(s)
if bucket_list:
event.add_bucket_list(bucket_list, analyst)
if ticket:
event.add_ticket(ticket, analyst)
try:
event.save(username=analyst)
# run event triage
event.reload()
run_triage(event, analyst)
message = ('<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' % (reverse('crits.events.views.view_event',
args=[event.id]),
title))
result = {'success': True,
'message': message,
'id': str(event.id)}
except ValidationError, e:
result = {'success': False,
'message': e}
def ip_add_update(ip_address,
ip_type,
source=None,
source_method='',
source_reference='',
campaign=None,
confidence='low',
analyst=None,
is_add_indicator=False,
indicator_reference='',
bucket_list=None,
ticket=None,
is_validate_only=False,
cache={},
id=None):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding/updating this IP.
:type analyst: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
if not source:
return {"success": False, "message": "Missing source information."}
(ip_address, error) = validate_and_normalize_ip(ip_address, ip_type)
if error:
return {"success": False, "message": error}
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if ip_object.check_message_received(id):
return {'success': False, 'message': 'dup'}
if isinstance(source, basestring):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)
]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
if bucket_list:
ip_object.add_bucket_list(bucket_list, analyst)
if ticket:
ip_object.add_ticket(ticket, analyst)
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=analyst)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' %
(resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
handle_indicator_ind(ip_address,
source,
ip_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
source_method,
indicator_reference,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(ip_object, analyst)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def handle_indicator_insert(
ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind["type"] not in IndicatorTypes.values():
return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
if ind["threat_type"] not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
if ind["attack_type"] not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}
(ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
if error:
return {"success": False, "message": error}
is_new_indicator = False
dmain = None
ip = None
rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}
if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
ind["status"] = Status.NEW
indicator = Indicator.objects(
ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind["type"]
indicator.threat_type = ind["threat_type"]
indicator.attack_type = ind["attack_type"]
indicator.value = ind["value"]
indicator.lower = ind["lower"]
indicator.description = ind["description"]
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
indicator.status = ind["status"]
is_new_indicator = True
else:
if ind["status"] != Status.NEW:
indicator.status = ind["status"]
add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
if indicator.description is None:
indicator.description = add_desc
else:
indicator.description += add_desc
if "campaign" in ind:
if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
confidence = ind.get("campaign_confidence", "low")
ind["campaign"] = EmbeddedCampaign(
name=ind["campaign"],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now(),
)
if isinstance(ind["campaign"], EmbeddedCampaign):
indicator.add_campaign(ind["campaign"])
elif isinstance(ind["campaign"], list):
for campaign in ind["campaign"]:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind["confidence"]
indicator.confidence.analyst = analyst
if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind["impact"]
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.lower
url_contains_ip = False
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
if ind_type == IndicatorTypes.URI:
domain_or_ip = urlparse.urlparse(ind_value).hostname
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
else:
domain_or_ip = ind_value
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(
domain_or_ip,
indicator.source,
username="******" % analyst,
campaign=indicator.campaign,
bucket_list=bucket_list,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success["object"]
if ind_type in IPTypes.values() or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = IndicatorTypes.IPV4_ADDRESS
except DjangoValidationError:
ind_type = IndicatorTypes.IPV6_ADDRESS
success = None
if add_domain:
success = ip_add_update(
ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success["object"]
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
ip.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
def add_new_actor(name,
aliases=None,
description=None,
source=None,
source_method='',
source_reference='',
campaign=None,
confidence=None,
analyst=None,
bucket_list=None,
ticket=None):
"""
Add an Actor to CRITs.
:param name: The name of the Actor.
:type name: str
:param aliases: Aliases for the actor.
:type aliases: list or str
:param description: Description of the actor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this actor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding this actor.
:type analyst: str
:param bucket_list: Buckets to assign to this actor.
:type bucket_list: str
:param ticket: Ticket to assign to this actor.
:type ticket: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.actors.actor.Actor`
"""
is_item_new = False
retVal = {}
actor = Actor.objects(name=name).first()
if not actor:
actor = Actor()
actor.name = name
if description:
actor.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)
]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
actor.add_campaign(camp)
if source:
for s in source:
actor.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
if not isinstance(aliases, list):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in actor.aliases:
actor.aliases.append(alias)
if bucket_list:
actor.add_bucket_list(bucket_list, analyst)
if ticket:
actor.add_ticket(ticket, analyst)
actor.save(username=analyst)
# run actor triage
if is_item_new:
actor.reload()
run_triage(actor, analyst)
resp_url = reverse('crits.actors.views.actor_detail', args=[actor.id])
retVal['message'] = ('Success! Click here to view the new Actor: '
'<a href="%s">%s</a>' % (resp_url, actor.name))
retVal['success'] = True
retVal['object'] = actor
retVal['id'] = str(actor.id)
return retVal
def add_new_exploit(name,
cve=None,
description=None,
source=None,
source_method=None,
source_reference=None,
campaign=None,
confidence=None,
user=None,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add an Exploit to CRITs.
:param name: The name of the exploit.
:type name: str
:param cve: CVE for the exploit.
:type cve: str
:param description: Description of the exploit.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this exploit.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this exploit.
:type user: str
:param bucket_list: Buckets to assign to this exploit.
:type bucket_list: str
:param ticket: Ticket to assign to this exploit.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.exploits.exploit.Exploit`
"""
is_item_new = False
retVal = {}
exploit = Exploit.objects(name=name).first()
if not exploit:
exploit = Exploit()
exploit.name = name
if description:
exploit.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=user)
]
elif isinstance(source, EmbeddedSource):
source = [source]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=user)
campaign = [c]
if campaign:
for camp in campaign:
exploit.add_campaign(camp)
if source:
for s in source:
exploit.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
exploit.cve = cve.strip()
if bucket_list:
exploit.add_bucket_list(bucket_list, user)
if ticket:
exploit.add_ticket(ticket, user)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
exploit.save(username=user)
if related_obj and exploit and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
exploit.add_relationship(related_obj,
relationship_type,
analyst=user,
get_rels=False)
exploit.save(username=user)
# run exploit triage
if is_item_new:
exploit.reload()
run_triage(exploit, user)
resp_url = reverse('crits.exploits.views.exploit_detail',
args=[exploit.id])
retVal['message'] = ('Success! Click here to view the new Exploit: '
'<a href="%s">%s</a>' % (resp_url, exploit.name))
retVal['success'] = True
retVal['object'] = exploit
retVal['id'] = str(exploit.id)
return retVal
def add_new_event(title, description, event_type, source, method, reference,
date, analyst, bucket_list=None, ticket=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param analyst: The user adding this Event.
:type analyst: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if not source:
return {'success': False, 'message': "Missing source information."}
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
s = create_embedded_source(name=source,
reference=reference,
method=method,
analyst=analyst,
date=date)
event.add_source(s)
if bucket_list:
event.add_bucket_list(bucket_list, analyst)
if ticket:
event.add_ticket(ticket, analyst)
try:
event.save(username=analyst)
# run event triage
event.reload()
run_triage(event, analyst)
message = ('<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' % (reverse('crits.events.views.view_event',
args=[event.id]),
title))
result = {'success': True,
'message': message,
'id': str(event.id),
'object': event}
except ValidationError, e:
result = {'success': False,
'message': e}
except Exception, e:
return {'success': False, 'message': e}
#Add relationships between fqdn, root
if fqdn_domain and root_domain:
root_domain.add_relationship(rel_item=fqdn_domain,
rel_type="Supra-domain_Of",
analyst=username,
get_rels=False)
root_domain.save(username=username)
fqdn_domain.save(username=username)
# run domain triage
if is_fqdn_domain_new:
fqdn_domain.reload()
run_triage(None, fqdn_domain, username)
if is_root_domain_new:
root_domain.reload()
run_triage(None, root_domain, username)
# return fqdn if they added an fqdn, or root if they added a root
if fqdn_domain:
return {'success': True, 'object': fqdn_domain, 'is_domain_new': is_fqdn_domain_new}
else:
return {'success': True, 'object': root_domain, 'is_domain_new': is_root_domain_new}
def update_tlds(data=None):
"""
Update the TLD list in the database.
:param data: The TLD data.
def handle_raw_data_file(data,
source_name,
user=None,
description=None,
title=None,
data_type=None,
tool_name=None,
tool_version=None,
tool_details=None,
link_id=None,
method=None,
reference=None,
copy_rels=False,
bucket_list=None,
ticket=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param reference: A reference to the source of this RawData.
:type reference: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success": False, "message": "Missing source information."}
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {'success': False, 'message': 'Invalid data type passed in'}
return status
data = data.encode('utf-8')
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if not raw_data:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
#raw_data.source = [source]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
source = create_embedded_source(source_name,
date=timestamp,
method=method,
reference=reference,
analyst=user)
# this will handle adding a new source, or an instance automatically
raw_data.add_source(source)
elif isinstance(source_name, EmbeddedSource):
raw_data.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
raw_data.add_source(s, method=method, reference=reference)
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
raw_data.add_relationship(
rel_id=rel.object_id,
type_=rel.rel_type,
rel_type=rel.relationship,
rel_date=rel.relationship_date,
analyst=user)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user)
# save raw_data
raw_data.save(username=user)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
'object': raw_data
}
return status
def ip_add_update(ip_address, ip_type, source=None, source_method=None,
source_reference=None, campaign=None, confidence='low', analyst=None,
is_add_indicator=False, indicator_reference=None,
bucket_list=None, ticket=None, is_validate_only=False, cache={}):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding/updating this IP.
:type analyst: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
if bucket_list:
ip_object.add_bucket_list(bucket_list, analyst)
if ticket:
ip_object.add_ticket(ticket, analyst)
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=analyst)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
handle_indicator_ind(ip_address,
source,
indicator_reference,
ip_type,
analyst,
source_method,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(None, ip_object, analyst)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def handle_pcap_file(
filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method=None,
relationship=None,
bucket_list=None,
ticket=None,
):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {"success": False, "message": "No data object passed in"}
return status
if len(data) <= 0:
status = {"success": False, "message": "Data length <= 0"}
return status
if (related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5)):
status = {"success": False, "message": "Must specify both related_type and related_id or related_md5."}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {"success": False, "message": "Related object not found."}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name, method=method, reference="", analyst=user)
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if not relationship:
relationship = "Related_To"
pcap.add_relationship(rel_item=related_obj, rel_type=relationship, analyst=user, get_rels=False)
related_obj.save(username=user)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(data, pcap, user)
status = {"success": True, "message": "Uploaded pcap", "md5": md5, "id": str(pcap.id)}
return status
def handle_pcap_file(filename, data, source_name, user=None,
description=None, related_id=None, related_md5=None,
related_type=None, method='', reference='', tlp='',
relationship=None, bucket_list=None, ticket=None):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param reference: A reference to the source of this PCAP.
:type reference: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {
'success': False,
'message': 'No data object passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if ((related_id or related_md5) and not related_type):
status = {
'success': False,
'message': 'Must specify both related_type and related_id or related_md5.'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {
'success': False,
'message': 'Related object not found.'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
method=method,
reference=reference,
tlp=tlp,
analyst=user.username)
else:
return {"success":False,
"message": "User does not have permission to add object \
using source %s." % source_name}
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s, method=method, reference=reference)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if relationship:
relationship=RelationshipTypes.inverse(relationship=relationship)
else:
relationship = RelationshipTypes.RELATED_TO
pcap.add_relationship(related_obj,
relationship,
analyst=user,
get_rels=False)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(pcap, user)
status = {
'success': True,
'message': 'Uploaded pcap',
'md5': md5,
'id': str(pcap.id),
'object': pcap
}
return status
def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if 'email_address' not in data:
return {'success': False, 'message': "No email address to look up"}
# check for exact match first
target = Target.objects(email_address=data['email_address']).first()
if not target: # if no exact match, look for case-insensitive match
target = Target.objects(
email_address__iexact=data['email_address']).first()
is_new = False
if not target:
is_new = True
target = Target()
target.email_address = data['email_address'].strip().lower()
bucket_list = False
ticket = False
if 'department' in data:
target.department = data['department']
if 'division' in data:
target.division = data['division']
if 'organization_id' in data:
target.organization_id = data['organization_id']
if 'firstname' in data:
target.firstname = data['firstname']
if 'lastname' in data:
target.lastname = data['lastname']
if 'note' in data:
target.note = data['note']
if 'title' in data:
target.title = data['title']
if 'campaign' in data and 'camp_conf' in data:
target.add_campaign(
EmbeddedCampaign(name=data['campaign'],
confidence=data['camp_conf'],
analyst=analyst))
if 'bucket_list' in data:
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if 'ticket' in data:
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
try:
target.save(username=analyst)
target.reload()
if is_new:
run_triage(target, analyst)
return {
'success': True,
'message': "Target saved successfully",
'id': str(target.id)
}
except ValidationError, e:
return {'success': False, 'message': "Target save failed: %s" % e}
def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if 'email_address' not in data:
return {'success': False,
'message': "No email address to look up"}
# check for exact match first
target = Target.objects(email_address=data['email_address']).first()
if not target: # if no exact match, look for case-insensitive match
target = Target.objects(email_address__iexact=data['email_address']).first()
is_new = False
if not target:
is_new = True
target = Target()
target.email_address = data['email_address'].strip().lower()
bucket_list = False
ticket = False
if 'department' in data:
target.department = data['department']
if 'division' in data:
target.division = data['division']
if 'organization_id' in data:
target.organization_id = data['organization_id']
if 'firstname' in data:
target.firstname = data['firstname']
if 'lastname' in data:
target.lastname = data['lastname']
if 'note' in data:
target.note = data['note']
if 'title' in data:
target.title = data['title']
if 'campaign' in data and 'camp_conf' in data:
target.add_campaign(EmbeddedCampaign(name=data['campaign'],
confidence=data['camp_conf'],
analyst=analyst))
if 'bucket_list' in data:
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if 'ticket' in data:
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if 'related_id' in data:
related_id = data['related_id']
if 'related_type' in data:
related_type = data['related_type']
if 'relationship_type' in data:
relationship_type = data['relationship_type']
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
target.save(username=analyst)
if related_obj and target:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
target.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
target.save(username=analyst)
target.reload()
if is_new:
run_triage(target, analyst)
return {'success': True,
'message': "Target saved successfully",
'id': str(target.id)}
except ValidationError, e:
return {'success': False,
'message': "Target save failed: %s" % e}
def handle_pcap_file(filename, data, source_name, user=None,
description=None, related_id=None, related_md5=None,
related_type=None, method='', reference='',
relationship=None, bucket_list=None, ticket=None):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param reference: A reference to the source of this PCAP.
:type reference: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {
'success': False,
'message': 'No data object passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if ((related_type and not (related_id or related_md5)) or
(not related_type and (related_id or related_md5))):
status = {
'success': False,
'message': 'Must specify both related_type and related_id or related_md5.'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {
'success': False,
'message': 'Related object not found.'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference=reference,
analyst=user)
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s, method=method, reference=reference)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if not relationship:
relationship = "Related_To"
pcap.add_relationship(related_obj,
relationship,
analyst=user,
get_rels=False)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(pcap, user)
status = {
'success': True,
'message': 'Uploaded pcap',
'md5': md5,
'id': str(pcap.id),
'object': pcap
}
return status
except Exception, e:
return {'success': False, 'message': e}
#Add relationships between fqdn, root
if fqdn_domain and root_domain:
root_domain.add_relationship(fqdn_domain,
"Supra-domain_Of",
analyst=username,
get_rels=False)
root_domain.save(username=username)
fqdn_domain.save(username=username)
# run domain triage
if is_fqdn_domain_new:
fqdn_domain.reload()
run_triage(fqdn_domain, username)
if is_root_domain_new:
root_domain.reload()
run_triage(root_domain, username)
# return fqdn if they added an fqdn, or root if they added a root
if fqdn_domain:
return {'success': True, 'object': fqdn_domain, 'is_domain_new': is_fqdn_domain_new}
else:
return {'success': True, 'object': root_domain, 'is_domain_new': is_root_domain_new}
def update_tlds(data=None):
"""
Update the TLD list in the database.
:param data: The TLD data.
def add_new_backdoor(name,
version=None,
aliases=None,
description=None,
source=None,
source_method=None,
source_reference=None,
source_tlp=None,
campaign=None,
confidence=None,
user=None,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add an Backdoor to CRITs.
:param name: The name of the backdoor.
:type name: str
:param version: Version of the backdoor.
:type version: str
:param aliases: Aliases for the backdoor.
:type aliases: list or str
:param description: Description of the backdoor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this backdoor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this backdoor.
:type user: str
:param bucket_list: Buckets to assign to this backdoor.
:type bucket_list: str
:param ticket: Ticket to assign to this backdoor.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"id" (str),
"object" (if successful) :class:`crits.backdoors.backdoor.Backdoor`
"""
retVal = {'success': False, 'message': ''}
username = user.username
if isinstance(source, basestring):
if user.check_source_write(source):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=username)
]
else:
return {
"success":
False,
"message":
"User does not have permission to add objects \
using source %s." % str(source)
}
elif isinstance(source, EmbeddedSource):
source = [source]
if not source:
retVal['message'] = "Missing source information."
return retVal
# When creating a backdoor object we can potentially create multiple
# objects. If we are given a name but no version we will create an object
# with just the name (called the "family backdoor"). If given a name and a
# version we will create the family backdoor and the specific backdoor for
# that given version.
# In case we create more than one backdoor object, store the created ones
# in this list. The list is walked later on and attributes applied to each
# object.
objs = []
# First check if we have the family (name and no version).
family = Backdoor.objects(name=name, version='').first()
if not family:
# Family does not exist, new object. Details are handled later.
family = Backdoor()
family.name = name
family.version = ''
objs.append(family)
# Now check if we have the specific instance for this name + version.
backdoor = None
if version:
backdoor = Backdoor.objects(name=name, version=version).first()
if not backdoor:
# Backdoor does not exist, new object. Details are handled later.
backdoor = Backdoor()
backdoor.name = name
backdoor.version = version
objs.append(backdoor)
# At this point we have a family object and potentially a specific object.
# Add the common parameters to all objects in the list and save them.
for backdoor in objs:
for s in source:
backdoor.add_source(s)
# Don't overwrite existing description.
if description and backdoor.description == '':
backdoor.description = description.strip()
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
if campaign:
for camp in campaign:
backdoor.add_campaign(camp)
if aliases:
if isinstance(aliases, basestring):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in backdoor.aliases:
backdoor.aliases.append(alias)
if bucket_list:
backdoor.add_bucket_list(bucket_list, username)
if ticket:
backdoor.add_ticket(ticket, username)
backdoor.save(username=username)
# run backdoor triage
backdoor.reload()
run_triage(backdoor, username)
# Because family objects are put in the list first we will always
# return a link to the most specific object created. If there is only
# one item in the list it will be the family object.
resp_url = reverse('crits.backdoors.views.backdoor_detail',
args=[backdoor.id])
retVal['message'] = 'Success: <a href="%s">%s</a>' % (resp_url,
backdoor.name)
retVal['object'] = backdoor
retVal['id'] = str(backdoor.id)
# Only relate to the most specific object created.
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
if related_obj and relationship_type and backdoor:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
backdoor.add_relationship(related_obj,
relationship_type,
analyst=username,
get_rels=False)
backdoor.save(username=username)
backdoor.reload()
# If we have a family and specific object, attempt to relate the two.
if len(objs) == 2:
objs[0].add_relationship(objs[1], RelationshipTypes.RELATED_TO)
objs[0].save()
retVal['success'] = True
return retVal
def handle_cert_file(filename, data, source_name, user=None,
description=None, related_id=None, related_md5=None,
related_type=None, method=None, relationship=None,
bucket_list=None, ticket=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_id: ObjectId of a top-level object related to this Certificate.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {
'success': False,
'message': 'No data object passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if ((related_type and not (related_id or related_md5)) or
(not related_type and (related_id or related_md5))):
status = {
'success': False,
'message': 'Must specify both related_type and related_id or related_md5.'
}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {
'success': False,
'message': 'Related object not found.'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference='',
analyst=user)
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if len(cert.analysis) < 1 and data:
run_triage(cert, user)
# update relationship if a related top-level object is supplied
if related_obj and cert:
if not relationship:
relationship = "Related_To"
cert.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
'id': str(cert.id)
}
return status
except Exception, e:
return {'success': False, 'message': e}
#Add relationships between fqdn, root
if fqdn_domain and root_domain:
root_domain.add_relationship(rel_item=fqdn_domain,
rel_type="Supra-domain_Of",
analyst=username,
get_rels=False)
root_domain.save(username=username)
fqdn_domain.save(username=username)
# run domain triage
if is_fqdn_domain_new:
fqdn_domain.reload()
run_triage(None, fqdn_domain, username)
if is_root_domain_new:
root_domain.reload()
run_triage(None, root_domain, username)
# return fqdn if they added an fqdn, or root if they added a root
if fqdn_domain:
return {'success': True, 'object': fqdn_domain, 'is_domain_new': is_fqdn_domain_new}
else:
return {'success': True, 'object': root_domain, 'is_domain_new': is_root_domain_new}
def update_tlds(data=None):
"""
Update the TLD list in the database.
:param data: The TLD data.
def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method=None, copy_rels=False,
bucket_list=None, ticket=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
data = data.encode('utf-8')
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# create source
source = create_embedded_source(source_name,
date=timestamp,
reference='',
method=method,
analyst=user)
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if raw_data:
raw_data.add_source(source)
else:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
raw_data.source = [source]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
raw_data.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type=rel.relationship,
rel_date=rel.relationship_date,
analyst=user)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
# save raw_data
raw_data.save(username=user)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
}
return status
fqdn_domain.add_relationship(related_obj,
relationship_type,
analyst=username,
get_rels=False)
fqdn_domain.save(username=username)
if root_domain and (related_obj != root_domain):
root_domain.add_relationship(related_obj,
relationship_type,
analyst=username,
get_rels=False)
root_domain.save(username=username)
# run domain triage
if is_fqdn_domain_new:
fqdn_domain.reload()
run_triage(fqdn_domain, username)
if is_root_domain_new:
root_domain.reload()
run_triage(root_domain, username)
# return fqdn if they added an fqdn, or root if they added a root
if fqdn_domain:
return {'success': True, 'object': fqdn_domain, 'is_domain_new': is_fqdn_domain_new}
else:
return {'success': True, 'object': root_domain, 'is_domain_new': is_root_domain_new}
def update_tlds(data=None):
"""
Update the TLD list in the database.
:param data: The TLD data.
def ip_add_update(ip_address, ip_type, source=None, source_method='',
source_reference='', source_tlp=None, campaign=None,
confidence='low', user=None, is_add_indicator=False,
indicator_reference='', bucket_list=None, ticket=None,
is_validate_only=False, cache={}, related_id=None,
related_type=None, relationship_type=None, description=''):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding/updating this IP.
:type user: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:param description: A description for this IP
:type description: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
if not source:
return {"success" : False, "message" : "Missing source information."}
source_name = source
(ip_address, error) = validate_and_normalize_ip(ip_address, ip_type)
if error:
return {"success": False, "message": error}
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if not ip_object.description:
ip_object.description = description or ''
elif ip_object.description != description:
ip_object.description += "\n" + (description or '')
if isinstance(source_name, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username)]
else:
return {"success":False,
"message": "User does not have permission to add object \
using source %s." % source}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user.username)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if bucket_list:
ip_object.add_bucket_list(bucket_list, user.username)
if ticket:
ip_object.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=user.username)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
result = handle_indicator_ind(ip_address,
source_name,
ip_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=source_method,
source_reference = indicator_reference,
source_tlp = source_tlp,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if related_obj and ip_object and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
ip_object.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
ip_object.save(username=user.username)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(ip_object, user)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def add_new_exploit(name, cve=None, description=None, source=None,
source_method=None, source_reference=None, campaign=None,
confidence=None, user=None, bucket_list=None,
ticket=None):
"""
Add an Exploit to CRITs.
:param name: The name of the exploit.
:type name: str
:param cve: CVE for the exploit.
:type cve: str
:param description: Description of the exploit.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this exploit.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this exploit.
:type user: str
:param bucket_list: Buckets to assign to this exploit.
:type bucket_list: str
:param ticket: Ticket to assign to this exploit.
:type ticket: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.exploits.exploit.Exploit`
"""
is_item_new = False
retVal = {}
exploit = Exploit.objects(name=name).first()
if not exploit:
exploit = Exploit()
exploit.name = name
if description:
exploit.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=user)]
elif isinstance(source, EmbeddedSource):
source = [source]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user)
campaign = [c]
if campaign:
for camp in campaign:
exploit.add_campaign(camp)
if source:
for s in source:
exploit.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
exploit.cve = cve.strip()
if bucket_list:
exploit.add_bucket_list(bucket_list, user)
if ticket:
exploit.add_ticket(ticket, user)
exploit.save(username=user)
# run exploit triage
if is_item_new:
exploit.reload()
run_triage(exploit, user)
resp_url = reverse('crits.exploits.views.exploit_detail', args=[exploit.id])
retVal['message'] = ('Success! Click here to view the new Exploit: '
'<a href="%s">%s</a>' % (resp_url, exploit.name))
retVal['success'] = True
retVal['object'] = exploit
retVal['id'] = str(exploit.id)
return retVal