def add_new_actor_identifier(identifier_type,
identifier=None,
source=None,
source_method='',
source_reference='',
analyst=None):
"""
Add an Actor Identifier to CRITs.
:param identifier_type: The Actor Identifier Type.
:type identifier_type: str
:param identifier: The Actor Identifier.
:type identifier: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param analyst: The user adding this actor.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
actor_identifier = ActorIdentifier.objects(identifier_type=identifier_type,
name=identifier).first()
if not actor_identifier:
actor_identifier = ActorIdentifier()
actor_identifier.set_identifier_type(identifier_type)
if not actor_identifier.identifier_type:
return {'success': False, 'message': "Unknown Identifier Type"}
if not identifier:
return {'success': False, 'message': "Missing Identifier"}
actor_identifier.name = identifier.strip()
if isinstance(source, basestring):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)
]
if source:
for s in source:
actor_identifier.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
actor_identifier.save(username=analyst)
actor_identifier.reload()
return {
'success': True,
'id': str(actor_identifier.id),
'message': "Actor Identifier added successfully!"
}
def add_new_actor_identifier(identifier_type, identifier=None, source=None,
source_method='', source_reference='',
source_tlp=None, user=None):
"""
Add an Actor Identifier to CRITs.
:param identifier_type: The Actor Identifier Type.
:type identifier_type: str
:param identifier: The Actor Identifier.
:type identifier: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param source_tlp: The TLP for this identifier.
:type source_tlp: str
:param user: The user adding this actor.
:type user: :class:`crits.core.user.CRITsUser`
:returns: dict with keys:
"success" (boolean),
"message" (str),
"""
actor_identifier = ActorIdentifier.objects(identifier_type=identifier_type,
name=identifier).first()
if not actor_identifier:
actor_identifier = ActorIdentifier()
actor_identifier.set_identifier_type(identifier_type)
if not actor_identifier.identifier_type:
return {'success': False,
'message': "Unknown Identifier Type"}
if not identifier:
return {'success': False,
'message': "Missing Identifier"}
actor_identifier.name = identifier.strip()
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username)]
if source:
for s in source:
actor_identifier.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
actor_identifier.save(username=user.username)
actor_identifier.reload()
return {'success': True,
'id': str(actor_identifier.id),
'message': "Actor Identifier added successfully!"}
def comment_add(cleaned_data, obj_type, obj_id, method, subscr, analyst):
"""
Add a new comment.
:param cleaned_data: Cleaned data from the Django form submission.
:type cleaned_data: dict
:param obj_type: The top-level object type to add the comment to.
:type obj_type: str
:param obj_id: The top-level ObjectId to add the comment to.
:type obj_id: str
:param method: If this is a reply or not (set method to "reply").
:type method: str
:param subscr: The subscription information for the top-level object.
:type subscr: dict
:param analyst: The user adding the comment.
:type analyst: str
:returns: dict with keys:
'success' (boolean),
'message': (str),
'html' (str) if successful.
"""
comment = Comment()
comment.comment = cleaned_data['comment']
comment.parse_comment()
comment.set_parent_object(obj_type, obj_id)
if method == "reply":
comment.set_parent_comment(cleaned_data['parent_date'],
cleaned_data['parent_analyst'])
comment.analyst = analyst
comment.set_url_key(cleaned_data['url_key'])
source = create_embedded_source(name=get_user_organization(analyst),
analyst=analyst,
needs_tlp=False)
comment.source = [source]
try:
comment.save(username=analyst)
# this is silly :( in the comment object the dates are still
# accurate to .###### seconds, but in the database are only
# accurate to .### seconds. This messes with the template's ability
# to compare creation and edit times.
comment.reload()
comment.comment_to_html()
html = render_to_string(
'comments_row_widget.html', {
'comment': comment,
'user': {
'username': analyst
},
'subscription': subscr
})
message = "Comment added successfully!"
result = {'success': True, 'html': html, 'message': message}
except ValidationError, e:
result = {'success': False, 'message': e}
def comment_add(cleaned_data, obj_type, obj_id, method, subscr, analyst):
"""
Add a new comment.
:param cleaned_data: Cleaned data from the Django form submission.
:type cleaned_data: dict
:param obj_type: The top-level object type to add the comment to.
:type obj_type: str
:param obj_id: The top-level ObjectId to add the comment to.
:type obj_id: str
:param method: If this is a reply or not (set method to "reply").
:type method: str
:param subscr: The subscription information for the top-level object.
:type subscr: dict
:param analyst: The user adding the comment.
:type analyst: str
:returns: dict with keys:
'success' (boolean),
'message': (str),
'html' (str) if successful.
"""
comment = Comment()
comment.comment = cleaned_data['comment']
comment.parse_comment()
comment.set_parent_object(obj_type, obj_id)
if method == "reply":
comment.set_parent_comment(cleaned_data['parent_date'],
cleaned_data['parent_analyst'])
comment.analyst = analyst
comment.set_url_key(cleaned_data['url_key'])
source = create_embedded_source(name=get_user_organization(analyst),
analyst=analyst, needs_tlp=False)
comment.source = [source]
try:
comment.save(username=analyst)
# this is silly :( in the comment object the dates are still
# accurate to .###### seconds, but in the database are only
# accurate to .### seconds. This messes with the template's ability
# to compare creation and edit times.
comment.reload()
comment.comment_to_html()
html = render_to_string('comments_row_widget.html',
{'comment': comment,
'user': {'username': analyst},
'subscription': subscr})
message = "Comment added successfully!"
result = {'success': True, 'html': html, 'message': message}
except ValidationError, e:
result = {'success': False, 'message': e}
def add_campaign(name,
description,
aliases,
analyst,
bucket_list=None,
ticket=None,
source=None,
id=None):
"""
Add a Campaign.
:param name: The name of the new Campaign.
:type name: str
:param description: Description of the new Campaign.
:type description: str
:param aliases: Aliases for the new Campaign.
:type aliases: str (comma separated) or list.
:param analyst: The user adding the Campaign.
:type analyst: str
:param bucket_list: Buckets to add to this Campaign.
:type bucket_list: str (comma separated) or list.
:param ticket: Ticket(s) to add to this Campaign.
:type ticket: str (comma separated) or list.
:returns: dict with key 'success' (boolean) and 'message' (str).
"""
# Verify the Campaign does not exist.
campaign = Campaign.objects(name=name).first()
if campaign:
if campaign.check_message_received(id):
return {'success': False, 'message': 'dup'}
return {
'success': False,
'message': ['Campaign already exists.'],
'id': str(campaign.id)
}
# Create new campaign.
campaign = Campaign(name=name)
campaign.edit_description(description)
if bucket_list:
campaign.add_bucket_list(bucket_list, analyst)
if ticket:
campaign.add_ticket(ticket, analyst)
if source:
source = create_embedded_source(name=source, analyst=analyst)
else:
source = create_embedded_source(name=get_user_organization(analyst),
analyst=analyst)
campaign.source = [source]
# Adjust aliases.
if isinstance(aliases, basestring):
alias_list = aliases.split(',')
final_aliases = [a.strip() for a in alias_list]
elif isinstance(aliases, list):
final_aliases = [a.strip() for a in aliases]
else:
final_aliases = []
campaign.add_alias(final_aliases)
try:
campaign.save(username=analyst)
campaign.reload()
return {
'success': True,
'message': 'Campaign created successfully!',
'id': str(campaign.id)
}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def add_new_exploit(name, cve=None, description=None, source=None,
source_method=None, source_reference=None, source_tlp=None,
campaign=None, confidence=None, user=None,
bucket_list=None, ticket=None,related_id=None,
related_type=None, relationship_type=None):
"""
Add an Exploit to CRITs.
:param name: The name of the exploit.
:type name: str
:param cve: CVE for the exploit.
:type cve: str
:param description: Description of the exploit.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param source_tlp: TLP value for this source
:type source_reference: str
:param campaign: A campaign to attribute to this exploit.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this exploit.
:type user: str
:param bucket_list: Buckets to assign to this exploit.
:type bucket_list: str
:param ticket: Ticket to assign to this exploit.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.exploits.exploit.Exploit`
"""
is_item_new = False
retVal = {}
exploit = Exploit.objects(name=name).first()
if not exploit:
exploit = Exploit()
exploit.name = name
if description:
exploit.description = description.strip()
is_item_new = True
username = user.username
if isinstance(source, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=username)]
else:
return {"success": False,
"message": "User does not have permission to add object \
using source %s." % source}
elif isinstance(source, EmbeddedSource):
source = [source]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
campaign = [c]
if campaign:
for camp in campaign:
exploit.add_campaign(camp)
if source:
for s in source:
exploit.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
exploit.cve = cve.strip()
if bucket_list:
exploit.add_bucket_list(bucket_list, user)
if ticket:
exploit.add_ticket(ticket, user)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
exploit.save(username=username)
if related_obj and exploit and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
exploit.add_relationship(related_obj,
relationship_type,
analyst=user,
get_rels=False)
exploit.save(username=username)
# run exploit triage
if is_item_new:
exploit.reload()
run_triage(exploit, username)
resp_url = reverse('crits-exploits-views-exploit_detail', args=[exploit.id])
retVal['message'] = ('Success! Click here to view the new Exploit: '
'<a href="%s">%s</a>' % (resp_url, exploit.name))
retVal['success'] = True
retVal['object'] = exploit
retVal['id'] = str(exploit.id)
return retVal
def handle_cert_file(filename,
data,
source_name,
user=None,
description=None,
related_md5=None,
method='',
reference='',
tlp=None,
relationship=None,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param reference: A reference to the source of this Certificate.
:type reference: str
:param tlp: The TLP for this certificate.
:type tlp: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
if ((related_type and not (related_id or related_md5))
or (not related_type and (related_id or related_md5))):
status = {
'success':
False,
'message':
'Must specify both related_type and related_id or related_md5.'
}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {'success': False, 'message': 'Related object not found.'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
reference=reference,
method=method,
tlp=tlp,
analyst=user.username)
else:
return {
"success":
False,
"message":
"User does not have permission to add objects \
using source %s." % str(source_name)
}
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name,
method=method,
reference=reference,
tlp=tlp)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s, method=method, reference=reference, tlp=tlp)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if AnalysisResult.objects(object_id=str(cert.id)).count() < 1 and data:
run_triage(cert, user)
# update relationship if a related top-level object is supplied
if related_obj and cert:
if relationship_type:
relationship = RelationshipTypes.inverse(
relationship=relationship_type)
if not relationship:
relationship = RelationshipTypes.RELATED_TO
cert.add_relationship(related_obj,
relationship,
analyst=user,
get_rels=False)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
'id': str(cert.id),
'object': cert
}
return status
def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method='', reference='', tlp='',
copy_rels=False, bucket_list=None, ticket=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param reference: A reference to the source of this RawData.
:type reference: str
:param tlp: TLP for the source.
:type tlp: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if isinstance(data, unicode):
data=data.encode('utf-8')
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if not raw_data:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
# raw_data.source = [source_name]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
source = create_embedded_source(source_name,
method=method,
reference=reference,
tlp=tlp,
analyst=user.username)
raw_data.add_source(source)
else:
return {"success":False,
"message": "User does not have permission to add object using source %s." % source_name}
# this will handle adding a new source, or an instance automatically
elif isinstance(source_name, EmbeddedSource):
raw_data.add_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.usrname)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
raw_data.add_source(s, method=method, reference=reference, tlp=tlp, analyst=user.username)
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
# Get object to relate to.
rel_item = class_from_id(rel.rel_type, rel.object_id)
if rel_item:
raw_data.add_relationship(rel_item,
rel.relationship,
rel_date=rel.relationship_date,
analyst=user.username)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
raw_data.save(username=user.username)
if related_obj and relationship_type and raw_data:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
raw_data.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
raw_data.save(username=user.username)
raw_data.reload()
# save raw_data
raw_data.save(username=user.username)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
'object': raw_data
}
return status
def add_new_domain(data,
request,
errors,
rowData=None,
is_validate_only=False,
cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: tuple (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
domain = data['domain']
add_ip = data.get('add_ip')
ip = data.get('ip')
ip_type = data.get('ip_type')
if add_ip:
error = validate_and_normalize_ip(ip, ip_type)[1]
if error:
errors.append(error)
if is_validate_only:
error = get_valid_root_domain(domain)[2]
if error:
errors.append(error)
# check for duplicate domains
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail',
args=[domain])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
result_cache[domain.lower()] = True
elif not errors:
username = request.user.username
reference = data.get('domain_reference')
source_name = data.get('domain_source')
method = data.get('domain_method')
source = [
create_embedded_source(source_name,
reference=reference,
method=method,
analyst=username)
]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [
EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)
]
else:
campaign = []
retVal = upsert_domain(domain,
source,
username,
campaign,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not retVal['success']:
errors.append(retVal.get('message'))
retVal['message'] = ""
else:
new_domain = retVal['object']
ip_result = {}
if add_ip:
if data.get('same_source'):
ip_source = source_name
ip_method = method
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(rel_item=new_ip,
rel_type='Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
new_ip.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail',
args=[domain])
if retVal['is_domain_new'] == True:
retVal['message'] = (
'Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, domain))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' %
(resp_url, domain))
retVal['message'] = message
retVal[form_consts.Status.
STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_tlo
# If we have an IP object, add an indicator for that.
if ip_result.get('success'):
ip = ip_result['object']
result = create_indicator_from_tlo('IP',
ip,
username,
ip_source,
add_domain=False)
ip_ind = result.get('indicator')
if not result['success']:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_tlo('Domain',
new_domain,
username,
source_name,
add_domain=False)
if not result['success']:
errors.append(result['message'])
elif ip_result.get('success') and ip_ind:
forge_relationship(left_class=result['indicator'],
right_class=ip_ind,
rel_type='Resolved_To',
analyst=username)
result = True
# This block validates, and may also add, objects to the Domain
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(domain, cache)
for object_row_counter, objectData in enumerate(
objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(
objectData, "Domain", current_domain.id)
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
current_domain = None
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
(obj_result, errors,
obj_retVal) = validate_and_add_new_handler_object(
None,
objectDict,
request,
errors,
object_row_counter,
is_validate_only=is_validate_only,
cache=cache,
obj=current_domain)
if not obj_result:
retVal['success'] = False
return result, errors, retVal
def add_screenshot(description, tags, source, method, reference, analyst,
screenshot, screenshot_ids, oid, otype):
"""
Add a screenshot or screenshots to a top-level object.
:param description: The description of the screenshot.
:type description: str
:param tags: Tags associated with this screenshot.
:type tags: str, list
:param source: The source who provided the screenshot.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param method: The method of acquiring this screenshot.
:type method: str
:param reference: A reference to the source of this screenshot.
:type reference: str
:param analyst: The user adding the screenshot.
:type analyst: str
:param screenshot: The screenshot to add.
:type screenshot: file handle
:param screenshot_ids: A list of ObjectIds of existing screenshots to add.
:type screenshot_ids: str, list
:param oid: The ObjectId of the top-level object to add to.
:type oid: str
:param otype: The top-level object type.
:type otype: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'id' (str) if successful,
'html' (str) if successful,
"""
result = {'success': False}
if not source:
result['message'] = "Must provide a source"
return result
obj = class_from_id(otype, oid)
if not obj:
result['message'] = "Could not find the top-level object."
return result
final_screenshots = []
if screenshot_ids:
if not isinstance(screenshot_ids, list):
screenshot_list = screenshot_ids.split(',')
else:
screenshot_list = screenshot_ids
for screenshot_id in screenshot_list:
screenshot_id = screenshot_id.strip().lower()
s = Screenshot.objects(id=screenshot_id).first()
if s:
if isinstance(source, basestring) and len(source) > 0:
s_embed = create_embedded_source(source,
method=method,
reference=reference,
analyst=analyst)
s.add_source(s_embed)
elif isinstance(source, EmbeddedSource):
s.add_source(source, method=method, reference=reference)
elif isinstance(source, list) and len(source) > 0:
for x in source:
if isinstance(x, EmbeddedSource):
s.add_source(x, method=method, reference=reference)
s.add_tags(tags)
s.save()
obj.screenshots.append(screenshot_id)
obj.save()
final_screenshots.append(s)
else:
md5 = hashlib.md5(screenshot.read()).hexdigest()
check = Screenshot.objects(md5=md5).first()
if check:
s = check
s.add_tags(tags)
else:
s = Screenshot()
s.analyst = analyst
s.description = description
s.md5 = md5
screenshot.seek(0)
s.add_screenshot(screenshot, tags)
if isinstance(source, basestring) and len(source) > 0:
s_embed = create_embedded_source(source,
method=method,
reference=reference,
analyst=analyst)
s.add_source(s_embed)
elif isinstance(source, EmbeddedSource):
s.add_source(source, method=method, reference=reference)
elif isinstance(source, list) and len(source) > 0:
for x in source:
if isinstance(x, EmbeddedSource):
s.add_source(x, method=method, reference=reference)
if not s.screenshot and not s.thumb:
result[
'message'] = "Problem adding screenshot to GridFS. No screenshot uploaded."
return result
try:
s.save(username=analyst)
final_screenshots.append(s)
except Exception, e:
result['message'] = str(e)
return result
obj.screenshots.append(str(s.id))
obj.save(username=analyst)
def add_new_actor(name, aliases=None, description=None, source=None,
source_method='', source_reference='', campaign=None,
confidence=None, analyst=None, bucket_list=None, ticket=None):
"""
Add an Actor to CRITs.
:param name: The name of the Actor.
:type name: str
:param aliases: Aliases for the actor.
:type aliases: list or str
:param description: Description of the actor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this actor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding this actor.
:type analyst: str
:param bucket_list: Buckets to assign to this actor.
:type bucket_list: str
:param ticket: Ticket to assign to this actor.
:type ticket: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.actors.actor.Actor`
"""
is_item_new = False
retVal = {}
actor = Actor.objects(name=name).first()
if not actor:
actor = Actor()
actor.name = name
if description:
actor.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
actor.add_campaign(camp)
if source:
for s in source:
actor.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if not isinstance(aliases, list):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in actor.aliases:
actor.aliases.append(alias)
if bucket_list:
actor.add_bucket_list(bucket_list, analyst)
if ticket:
actor.add_ticket(ticket, analyst)
actor.save(username=analyst)
# run actor triage
if is_item_new:
actor.reload()
run_triage(actor, analyst)
resp_url = reverse('crits.actors.views.actor_detail', args=[actor.id])
retVal['message'] = ('Success! Click here to view the new Actor: '
'<a href="%s">%s</a>' % (resp_url, actor.name))
retVal['success'] = True
retVal['object'] = actor
retVal['id'] = str(actor.id)
return retVal
def handle_pcap_file(filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method='',
reference='',
relationship=None,
bucket_list=None,
ticket=None):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param reference: A reference to the source of this PCAP.
:type reference: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
if ((related_type and not (related_id or related_md5))
or (not related_type and (related_id or related_md5))):
status = {
'success':
False,
'message':
'Must specify both related_type and related_id or related_md5.'
}
return status
if not source_name:
return {"success": False, "message": "Missing source information."}
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {'success': False, 'message': 'Related object not found.'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference=reference,
analyst=user)
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s, method=method, reference=reference)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if not relationship:
relationship = "Related_To"
pcap.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(pcap, user)
status = {
'success': True,
'message': 'Uploaded pcap',
'md5': md5,
'id': str(pcap.id),
'object': pcap
}
return status
def add_screenshot(description, tags, source, method, reference, tlp, analyst,
screenshot, screenshot_ids, oid, otype):
"""
Add a screenshot or screenshots to a top-level object.
:param description: The description of the screenshot.
:type description: str
:param tags: Tags associated with this screenshot.
:type tags: str, list
:param source: The source who provided the screenshot.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param method: The method of acquiring this screenshot.
:type method: str
:param reference: A reference to the source of this screenshot.
:type reference: str
:param tlp: The TLP Sharing of this screenshot.
:type tlp: str
:param analyst: The user adding the screenshot.
:type analyst: str
:param screenshot: The screenshot to add.
:type screenshot: file handle
:param screenshot_ids: A list of ObjectIds of existing screenshots to add.
:type screenshot_ids: str, list
:param oid: The ObjectId of the top-level object to add to.
:type oid: str
:param otype: The top-level object type.
:type otype: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'id' (str) if successful,
'html' (str) if successful,
"""
result = {'success': False}
if not source:
result['message'] = "Must provide a source"
return result
obj = class_from_id(otype, oid)
if not obj:
result['message'] = "Could not find the top-level object."
return result
final_screenshots = []
if screenshot_ids:
if not isinstance(screenshot_ids, list):
screenshot_list = screenshot_ids.split(',')
else:
screenshot_list = screenshot_ids
for screenshot_id in screenshot_list:
screenshot_id = screenshot_id.strip().lower()
s = Screenshot.objects(id=screenshot_id).first()
if s:
if isinstance(source, basestring) and len(source) > 0:
s_embed = create_embedded_source(source, method=method,
reference=reference,
analyst=analyst,
tlp=tlp)
s.add_source(s_embed)
elif isinstance(source, EmbeddedSource):
s.add_source(source=source, method=method,
reference=reference, analyst=analyst, tlp=tlp)
elif isinstance(source, list) and len(source) > 0:
for x in source:
if isinstance(x, EmbeddedSource):
s.add_source(x, method=method, reference=reference,
analyst=analyst, tlp=tlp)
s.add_tags(tags)
s.save()
obj.screenshots.append(screenshot_id)
obj.save()
final_screenshots.append(s)
else:
md5 = hashlib.md5(screenshot.read()).hexdigest()
check = Screenshot.objects(md5=md5).first()
if check:
s = check
s.add_tags(tags)
else:
s = Screenshot()
s.analyst = analyst
s.description = description
s.md5 = md5
screenshot.seek(0)
s.add_screenshot(screenshot, tags)
if isinstance(source, basestring) and len(source) > 0:
s_embed = create_embedded_source(source, method=method,
reference=reference,
analyst=analyst,
tlp=tlp)
s.add_source(s_embed)
elif isinstance(source, EmbeddedSource):
s.add_source(source, method=method, reference=reference,
analyst=analyst, tlp=tlp)
elif isinstance(source, list) and len(source) > 0:
for x in source:
if isinstance(x, EmbeddedSource):
s.add_source(x, method=method, reference=reference,
analyst=analyst, tlp=tlp)
if not s.screenshot and not s.thumb:
result['message'] = "Problem adding screenshot to GridFS. No screenshot uploaded."
return result
try:
s.save(username=analyst)
final_screenshots.append(s)
except Exception, e:
result['message'] = str(e)
return result
obj.screenshots.append(str(s.id))
obj.save(username=analyst)
def add_new_event(title,
description,
event_type,
source,
method,
reference,
date,
analyst,
bucket_list=None,
ticket=None,
campaign=None,
campaign_confidence=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param analyst: The user adding this Event.
:type analyst: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys "success" (boolean) and "message" (str)
:param campaign: Campaign to associate with this Event
:type campaign: str
:param campaign_confidence: Confidence to associate with the Campaign
:type campaign_confidence: str
"""
result = dict()
if not source:
return {'success': False, 'message': "Missing source information."}
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
s = create_embedded_source(name=source,
reference=reference,
method=method,
analyst=analyst,
date=date)
event.add_source(s)
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'
}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
if campaign:
if isinstance(campaign, basestring) and len(campaign) > 0:
if campaign.lower() not in valid_campaigns:
result = {
'success': False,
'message': '{} is not a valid campaign.'.format(campaign)
}
else:
confidence = valid_campaign_confidence.get(
campaign_confidence, 'low')
campaign = EmbeddedCampaign(name=campaign,
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
event.add_campaign(campaign)
if bucket_list:
event.add_bucket_list(bucket_list, analyst)
if ticket:
event.add_ticket(ticket, analyst)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
event.save(username=analyst)
if related_obj and event and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
event.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
event.save(username=analyst)
# run event triage
event.reload()
run_triage(event, analyst)
message = (
'<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' %
(reverse('crits.events.views.view_event', args=[event.id]), title))
result = {
'success': True,
'message': message,
'id': str(event.id),
'object': event
}
except ValidationError, e:
result = {'success': False, 'message': e}
def ip_add_update(ip_address,
ip_type,
source=None,
source_method='',
source_reference='',
source_tlp=None,
campaign=None,
confidence='low',
user=None,
is_add_indicator=False,
indicator_reference='',
bucket_list=None,
ticket=None,
is_validate_only=False,
cache={},
related_id=None,
related_type=None,
relationship_type=None,
description=''):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding/updating this IP.
:type user: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:param description: A description for this IP
:type description: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
if not source:
return {"success": False, "message": "Missing source information."}
source_name = source
(ip_address, error) = validate_and_normalize_ip(ip_address, ip_type)
if error:
return {"success": False, "message": error}
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if not ip_object.description:
ip_object.description = description or ''
elif ip_object.description != description:
ip_object.description += "\n" + (description or '')
if isinstance(source_name, basestring):
if user.check_source_write(source):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username)
]
else:
return {
"success":
False,
"message":
"User does not have permission to add object \
using source %s." % source
}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=user.username)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
if bucket_list:
ip_object.add_bucket_list(bucket_list, user.username)
if ticket:
ip_object.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=user.username)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' %
(resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
result = handle_indicator_ind(ip_address,
source_name,
ip_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=source_method,
source_reference=indicator_reference,
source_tlp=source_tlp,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if related_obj and ip_object and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
ip_object.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
ip_object.save(username=user.username)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(ip_object, user)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: tuple
"""
username = request.user.username
result = False
retVal = {}
reference = data.get('domain_reference')
name = data.get('domain_source')
method = data.get('domain_method')
source = [create_embedded_source(name, reference=reference, method=method,
analyst=username)]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)]
else:
campaign = []
(sdomain, fqdn) = get_domain(data['domain'])
if sdomain == "no_tld_found_error":
errors.append(u"Error: Invalid domain: " + data['domain'])
elif is_validate_only == False:
retVal = upsert_domain(sdomain, fqdn, source, username, campaign,
bucket_list=bucket_list, ticket=ticket, cache=cache)
ip_result = None
if retVal['success']:
add_ip = data.get('add_ip')
if add_ip:
ip = data.get('ip')
if data.get('same_source'):
ip_source = data.get('domain_source')
ip_method = data.get('domain_method')
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
None,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_domain = retVal['object']
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(rel_item=new_ip,
rel_type='Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
new_ip.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn])
if retVal['is_domain_new'] == True:
retVal['message'] = ('Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, fqdn))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, fqdn))
retVal['message'] = message
retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
# add_indicators_for_domain handles adding relationships as well
if data.get('add_indicators'):
analyst = username
from crits.indicators.handlers import add_indicators_for_domain
if ip_result and ip_result['success']:
obj = ip_result['object']
errors += add_indicators_for_domain(sdomain,
fqdn,
source,
analyst,
reference,
obj,
bucket_list=bucket_list,
ticket=ticket)['errors']
else:
errors += add_indicators_for_domain(sdomain,
fqdn,
source,
analyst,
reference,
bucket_list=bucket_list,
ticket=ticket)['errors']
result = True
elif 'message' in retVal: #database error? (!c_dom)
errors.append(retVal['message']) #u"Unknown error: unable to add domain")
elif is_validate_only == True:
domain = data['domain']
fqdn_domain = retrieve_domain(domain, cache);
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, fqdn))
retVal['message'] = message;
retVal['status'] = form_consts.Status.DUPLICATE;
retVal['warning'] = message;
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
result_cache[domain.lower()] = True;
# This block tries to add objects to the item
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
object_row_counter = 1
current_domain = retrieve_domain(fqdn, cache)
for objectData in objectsData:
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(objectData, "Domain", current_domain.id)
else:
objectDict = object_array_to_dict(objectData, "Domain", "")
current_domain = None;
else:
objectDict = object_array_to_dict(objectData, "Domain", "")
(object_result, object_errors, object_retVal) = validate_and_add_new_handler_object(
None, objectDict, request, errors, object_row_counter,
is_validate_only=is_validate_only,
cache=cache, obj=current_domain)
if object_retVal.get('success') == False:
retVal['success'] = False
if object_retVal.get('message'):
errors.append(object_retVal['message'])
object_row_counter += 1
return result, errors, retVal
def handle_cert_file(filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method=None,
relationship=None,
bucket_list=None,
ticket=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_id: ObjectId of a top-level object related to this Certificate.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference='',
analyst=user)
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if len(cert.analysis) < 1 and data:
run_triage(data, cert, user)
# update relationship if a related top-level object is supplied
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if related_obj and cert:
if not relationship:
relationship = "Related_To"
cert.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
}
return status
def handle_pcap_file(
filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method=None,
relationship=None,
bucket_list=None,
ticket=None,
):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {"success": False, "message": "No data object passed in"}
return status
if len(data) <= 0:
status = {"success": False, "message": "Data length <= 0"}
return status
if (related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5)):
status = {"success": False, "message": "Must specify both related_type and related_id or related_md5."}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {"success": False, "message": "Related object not found."}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name, method=method, reference="", analyst=user)
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if not relationship:
relationship = "Related_To"
pcap.add_relationship(rel_item=related_obj, rel_type=relationship, analyst=user, get_rels=False)
related_obj.save(username=user)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(data, pcap, user)
status = {"success": True, "message": "Uploaded pcap", "md5": md5, "id": str(pcap.id)}
return status
def add_new_actor(name, aliases=None, description=None, source=None,
source_method='', source_reference='', source_tlp=None,
campaign=None, confidence=None, user=None,
bucket_list=None, ticket=None, related_id=None,
related_type=None, relationship_type=None):
"""
Add an Actor to CRITs.
:param name: The name of the Actor.
:type name: str
:param aliases: Aliases for the actor.
:type aliases: list or str
:param description: Description of the actor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param source_tlp: The TLP for this Actor.
:type source_tlp: str
:param campaign: A campaign to attribute to this actor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this actor.
:type user: :class:`crits.core.user.CRITsUser`
:param bucket_list: Buckets to assign to this actor.
:type bucket_list: str
:param ticket: Ticket to assign to this actor.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.actors.actor.Actor`
"""
username = user.username
is_item_new = False
retVal = {}
actor = Actor.objects(name=name).first()
if not actor:
actor = Actor()
actor.name = name
if description:
actor.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=username)]
else:
return {"success": False,
"message": "User does not have permission to add objects \
using source %s." % str(source)}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
if campaign:
for camp in campaign:
actor.add_campaign(camp)
if source:
for s in source:
actor.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if not isinstance(aliases, list):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in actor.aliases:
actor.aliases.append(alias)
if bucket_list:
actor.add_bucket_list(bucket_list, username)
if ticket:
actor.add_ticket(ticket, username)
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
actor.save(username=username)
if related_obj and actor:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
actor.add_relationship(related_obj,
relationship_type,
analyst=username,
get_rels=False)
actor.save(username=username)
actor.reload()
# run actor triage
if is_item_new:
actor.reload()
run_triage(actor, user)
resp_url = reverse('crits.actors.views.actor_detail', args=[actor.id])
retVal['message'] = ('Success! Click here to view the new Actor: '
'<a href="%s">%s</a>' % (resp_url, actor.name))
retVal['success'] = True
retVal['object'] = actor
retVal['id'] = str(actor.id)
return retVal
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: tuple (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
domain = data['domain']
add_ip = data.get('add_ip')
ip = data.get('ip')
ip_type = data.get('ip_type')
if add_ip:
error = validate_and_normalize_ip(ip, ip_type)[1]
if error:
errors.append(error)
if is_validate_only:
error = get_valid_root_domain(domain)[2]
if error:
errors.append(error)
# check for duplicate domains
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
result_cache[domain.lower()] = True
elif not errors:
username = request.user.username
reference = data.get('domain_reference')
source_name = data.get('domain_source')
method = data.get('domain_method')
source = [create_embedded_source(source_name, reference=reference,
method=method, analyst=username)]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)]
else:
campaign = []
retVal = upsert_domain(domain, source, username, campaign,
bucket_list=bucket_list, ticket=ticket, cache=cache)
if not retVal['success']:
errors.append(retVal.get('message'))
retVal['message'] = ""
else:
new_domain = retVal['object']
ip_result = {}
if add_ip:
if data.get('same_source'):
ip_source = source_name
ip_method = method
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(new_ip,
'Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
if retVal['is_domain_new'] == True:
retVal['message'] = ('Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, domain))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_tlo
# If we have an IP object, add an indicator for that.
if ip_result.get('success'):
ip = ip_result['object']
result = create_indicator_from_tlo('IP',
ip,
username,
ip_source,
add_domain=False)
ip_ind = result.get('indicator')
if not result['success']:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_tlo('Domain',
new_domain,
username,
source_name,
add_domain=False)
if not result['success']:
errors.append(result['message'])
elif ip_result.get('success') and ip_ind:
forge_relationship(left_class=result['indicator'],
right_class=ip_ind,
rel_type='Resolved_To',
analyst=username)
result = True
# This block validates, and may also add, objects to the Domain
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(domain, cache)
for object_row_counter, objectData in enumerate(objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(objectData,
"Domain",
current_domain.id)
else:
objectDict = object_array_to_dict(objectData,
"Domain",
"")
current_domain = None;
else:
objectDict = object_array_to_dict(objectData,
"Domain",
"")
(obj_result,
errors,
obj_retVal) = validate_and_add_new_handler_object(
None, objectDict, request, errors, object_row_counter,
is_validate_only=is_validate_only,
cache=cache, obj=current_domain)
if not obj_result:
retVal['success'] = False
return result, errors, retVal
def ip_add_update(ip_address, ip_type, source=None, source_method=None,
source_reference=None, campaign=None, confidence='low', analyst=None,
is_add_indicator=False, indicator_reference=None,
bucket_list=None, ticket=None, is_validate_only=False, cache={}):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding/updating this IP.
:type analyst: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
if bucket_list:
ip_object.add_bucket_list(bucket_list, analyst)
if ticket:
ip_object.add_ticket(ticket, analyst)
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=analyst)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
handle_indicator_ind(ip_address,
source,
indicator_reference,
ip_type,
analyst,
source_method,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(None, ip_object, analyst)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def handle_signature_file(data,
source_name,
user=None,
description=None,
title=None,
data_type=None,
data_type_min_version=None,
data_type_max_version=None,
data_type_dependency=None,
link_id=None,
source_method='',
source_reference='',
source_tlp='',
copy_rels=False,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add Signature.
:param data: The data of the Signature.
:type data: str
:param source_name: The source which provided this Signature.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Signature.
:type user: str
:param description: Description of the Signature.
:type description: str
:param title: Title of the Signature.
:type title: str
:param data_type: Datatype of the Signature.
:type data_type: str
:param data_type: Datatype of the Signature.
:type data_type_min_version: str
:param data_type_min_version: Datatype tool minimum version.
:type data_type_max_version: str
:param data_type_max_version: Datatype tool maximum version.
:type data_type_dependency: list
:param data_type_dependency: Datatype tool dependency to be run
:param link_id: LinkId to tie this to another Signature as a new version.
:type link_id: str
:param method: The method of acquiring this Signature.
:type method: str
:param reference: A reference to the source of this Signature.
:type reference: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this Signature
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Signature
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success": False, "message": "Missing source information."}
rdt = SignatureType.objects(name=data_type).first()
if not rdt:
status = {'success': False, 'message': 'Invalid data type passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate signature
signature = Signature()
signature.created = timestamp
signature.description = description
signature.md5 = md5
signature.data = data
signature.title = title
signature.data_type = data_type
signature.data_type_min_version = data_type_min_version
signature.data_type_max_version = data_type_max_version
if data_type_dependency:
if type(data_type_dependency) == unicode:
data_type_dependency = data_type_dependency.split(",")
for item in data_type_dependency:
if item:
item = item.strip()
signature.data_type_dependency.append(str(item))
else:
data_type_dependency = []
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
source = create_embedded_source(source_name,
date=timestamp,
method=source_method,
reference=source_reference,
tlp=source_tlp,
analyst=user.username)
# this will handle adding a new source, or an instance automatically
signature.add_source(source)
else:
return {
"success":
False,
"message":
"User does not have permission to add object using source %s."
% source_name
}
elif isinstance(source_name, EmbeddedSource):
signature.add_source(source_name,
method=source_method,
reference=source_reference,
tlp=source_tlp)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
signature.add_source(s,
method=source_method,
reference=source_reference,
source_tlp=source_tlp)
signature.version = len(Signature.objects(link_id=link_id)) + 1
if link_id:
signature.link_id = link_id
if copy_rels:
rd2 = Signature.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
signature.save(username=user.username)
signature.reload()
for rel in rd2.relationships:
# Get object to relate to.
rel_item = class_from_id(rel.rel_type, rel.object_id)
if rel_item:
signature.add_relationship(
rel_item,
rel.relationship,
rel_date=rel.relationship_date,
analyst=user.username)
if bucket_list:
signature.add_bucket_list(bucket_list, user)
if ticket:
signature.add_ticket(ticket, user)
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
status = {'success': False, 'message': 'Related Object not found.'}
return status
signature.save(username=user.username)
if related_obj and signature and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
signature.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
signature.save(username=user.username)
signature.reload()
# save signature
signature.save(username=user.username)
signature.reload()
status = {
'success': True,
'message': 'Uploaded signature',
'_id': signature.id,
'object': signature
}
return status
def handle_signature_file(data, source_name, user=None,
description=None, title=None, data_type=None,
data_type_min_version=None, data_type_max_version=None,
data_type_dependency=None, link_id=None, method='', reference='',
copy_rels=False, bucket_list=None, ticket=None):
"""
Add Signature.
:param data: The data of the Signature.
:type data: str
:param source_name: The source which provided this Signature.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Signature.
:type user: str
:param description: Description of the Signature.
:type description: str
:param title: Title of the Signature.
:type title: str
:param data_type: Datatype of the Signature.
:type data_type: str
:param data_type: Datatype of the Signature.
:type data_type_min_version: str
:param data_type_min_version: Datatype tool minimum version.
:type data_type_max_version: str
:param data_type_max_version: Datatype tool maximum version.
:type data_type_dependency: list
:param data_type_dependency: Datatype tool dependency to be run
:param link_id: LinkId to tie this to another Signature as a new version.
:type link_id: str
:param method: The method of acquiring this Signature.
:type method: str
:param reference: A reference to the source of this Signature.
:type reference: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this Signature
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Signature
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
rdt = SignatureType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate signature
signature = Signature()
signature.created = timestamp
signature.description = description
signature.md5 = md5
signature.data = data
signature.title = title
signature.data_type = data_type
signature.data_type_min_version = data_type_min_version
signature.data_type_max_version = data_type_max_version
if data_type_dependency:
if type(data_type_dependency) == unicode:
data_type_dependency = data_type_dependency.split(",")
for item in data_type_dependency:
if item:
item = item.strip()
signature.data_type_dependency.append(str(item))
else:
data_type_dependency = []
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
source = create_embedded_source(source_name,
date=timestamp,
method=method,
reference=reference,
analyst=user)
# this will handle adding a new source, or an instance automatically
signature.add_source(source)
elif isinstance(source_name, EmbeddedSource):
signature.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
signature.add_source(s, method=method, reference=reference)
signature.version = len(Signature.objects(link_id=link_id)) + 1
if link_id:
signature.link_id = link_id
if copy_rels:
rd2 = Signature.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
signature.save(username=user)
signature.reload()
for rel in rd2.relationships:
# Get object to relate to.
rel_item = class_from_id(rel.rel_type, rel.object_id)
if rel_item:
signature.add_relationship(rel_item,
rel.relationship,
rel_date=rel.relationship_date,
analyst=user)
if bucket_list:
signature.add_bucket_list(bucket_list, user)
if ticket:
signature.add_ticket(ticket, user);
# save signature
signature.save(username=user)
signature.reload()
status = {
'success': True,
'message': 'Uploaded signature',
'_id': signature.id,
'object': signature
}
return status
def handle_cert_file(filename, data, source_name, user=None,
description=None, related_id=None, related_md5=None,
related_type=None, method=None, relationship=None,
bucket_list=None, ticket=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_id: ObjectId of a top-level object related to this Certificate.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {
'success': False,
'message': 'No data object passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if ((related_type and not (related_id or related_md5)) or
(not related_type and (related_id or related_md5))):
status = {
'success': False,
'message': 'Must specify both related_type and related_id or related_md5.'
}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {
'success': False,
'message': 'Related object not found.'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference='',
analyst=user)
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if len(cert.analysis) < 1 and data:
run_triage(cert, user)
# update relationship if a related top-level object is supplied
if related_obj and cert:
if not relationship:
relationship = "Related_To"
cert.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
'id': str(cert.id)
}
return status
def add_new_exploit(name,
cve=None,
description=None,
source=None,
source_method=None,
source_reference=None,
source_tlp=None,
campaign=None,
confidence=None,
user=None,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add an Exploit to CRITs.
:param name: The name of the exploit.
:type name: str
:param cve: CVE for the exploit.
:type cve: str
:param description: Description of the exploit.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param source_tlp: TLP value for this source
:type source_reference: str
:param campaign: A campaign to attribute to this exploit.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this exploit.
:type user: str
:param bucket_list: Buckets to assign to this exploit.
:type bucket_list: str
:param ticket: Ticket to assign to this exploit.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.exploits.exploit.Exploit`
"""
is_item_new = False
retVal = {}
exploit = Exploit.objects(name=name).first()
if not exploit:
exploit = Exploit()
exploit.name = name
if description:
exploit.description = description.strip()
is_item_new = True
username = user.username
if isinstance(source, basestring):
if user.check_source_write(source):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=username)
]
else:
return {
"success":
False,
"message":
"User does not have permission to add object \
using source %s." % source
}
elif isinstance(source, EmbeddedSource):
source = [source]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
if campaign:
for camp in campaign:
exploit.add_campaign(camp)
if source:
for s in source:
exploit.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
exploit.cve = cve.strip()
if bucket_list:
exploit.add_bucket_list(bucket_list, user)
if ticket:
exploit.add_ticket(ticket, user)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
exploit.save(username=username)
if related_obj and exploit and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
exploit.add_relationship(related_obj,
relationship_type,
analyst=user,
get_rels=False)
exploit.save(username=username)
# run exploit triage
if is_item_new:
exploit.reload()
run_triage(exploit, username)
resp_url = reverse('crits.exploits.views.exploit_detail',
args=[exploit.id])
retVal['message'] = ('Success! Click here to view the new Exploit: '
'<a href="%s">%s</a>' % (resp_url, exploit.name))
retVal['success'] = True
retVal['object'] = exploit
retVal['id'] = str(exploit.id)
return retVal
def add_new_event(title, description, event_type, source, method, reference,
date, analyst, bucket_list=None, ticket=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param analyst: The user adding this Event.
:type analyst: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if not source:
return {'success': False, 'message': "Missing source information."}
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
s = create_embedded_source(name=source,
reference=reference,
method=method,
analyst=analyst,
date=date)
event.add_source(s)
if bucket_list:
event.add_bucket_list(bucket_list, analyst)
if ticket:
event.add_ticket(ticket, analyst)
try:
event.save(username=analyst)
# run event triage
event.reload()
run_triage(event, analyst)
message = ('<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' % (reverse('crits.events.views.view_event',
args=[event.id]),
title))
result = {'success': True,
'message': message,
'id': str(event.id)}
except ValidationError, e:
result = {'success': False,
'message': e}
def add_new_event(title, description, event_type, source_name, source_method,
source_reference, source_tlp, date, user,
bucket_list=None, ticket=None, campaign=None, campaign_confidence=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param user: The user adding this Event.
:type user: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys "success" (boolean) and "message" (str)
:param campaign: Campaign to associate with this Event
:type campaign: str
:param campaign_confidence: Confidence to associate with the Campaign
:type campaign_confidence: str
"""
if not source_name:
return {'success': False, 'message': "Missing source information."}
result = dict()
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username,
date=date)
else:
return {"success": False,
"message": "User does not have permission to add object \
using source %s." % source_name}
event.add_source(s)
valid_campaign_confidence = {
'low': 'low',
'medium': 'medium',
'high': 'high'}
valid_campaigns = {}
for c in Campaign.objects(active='on'):
valid_campaigns[c['name'].lower()] = c['name']
if campaign:
if isinstance(campaign, basestring) and len(campaign) > 0:
if campaign.lower() not in valid_campaigns:
result = {'success':False, 'message':'{} is not a valid campaign.'.format(campaign)}
else:
confidence = valid_campaign_confidence.get(campaign_confidence, 'low')
campaign = EmbeddedCampaign(name=campaign,
confidence=confidence,
description="",
analyst=user.username,
date=datetime.datetime.now())
event.add_campaign(campaign)
if bucket_list:
event.add_bucket_list(bucket_list, user.username)
if ticket:
event.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
event.save(username=user.username)
if related_obj and event and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
event.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
event.save(username=user.username)
# run event triage
event.reload()
run_triage(event, user.username)
message = ('<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' % (reverse('crits-events-views-view_event',
args=[event.id]),
title))
result = {'success': True,
'message': message,
'id': str(event.id),
'object': event}
except ValidationError, e:
result = {'success': False,
'message': e}
def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method=None, copy_rels=False,
bucket_list=None, ticket=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
data = data.encode('utf-8')
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# create source
source = create_embedded_source(source_name,
date=timestamp,
reference='',
method=method,
analyst=user)
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if raw_data:
raw_data.add_source(source)
else:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
raw_data.source = [source]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
raw_data.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type=rel.relationship,
rel_date=rel.relationship_date,
analyst=user)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
# save raw_data
raw_data.save(username=user)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
}
return status
def add_new_actor(name,
aliases=None,
description=None,
source=None,
source_method='',
source_reference='',
campaign=None,
confidence=None,
analyst=None,
bucket_list=None,
ticket=None):
"""
Add an Actor to CRITs.
:param name: The name of the Actor.
:type name: str
:param aliases: Aliases for the actor.
:type aliases: list or str
:param description: Description of the actor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this actor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding this actor.
:type analyst: str
:param bucket_list: Buckets to assign to this actor.
:type bucket_list: str
:param ticket: Ticket to assign to this actor.
:type ticket: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.actors.actor.Actor`
"""
is_item_new = False
retVal = {}
actor = Actor.objects(name=name).first()
if not actor:
actor = Actor()
actor.name = name
if description:
actor.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)
]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
actor.add_campaign(camp)
if source:
for s in source:
actor.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
if not isinstance(aliases, list):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in actor.aliases:
actor.aliases.append(alias)
if bucket_list:
actor.add_bucket_list(bucket_list, analyst)
if ticket:
actor.add_ticket(ticket, analyst)
actor.save(username=analyst)
# run actor triage
if is_item_new:
actor.reload()
run_triage(actor, analyst)
resp_url = reverse('crits.actors.views.actor_detail', args=[actor.id])
retVal['message'] = ('Success! Click here to view the new Actor: '
'<a href="%s">%s</a>' % (resp_url, actor.name))
retVal['success'] = True
retVal['object'] = actor
retVal['id'] = str(actor.id)
return retVal
def add_new_event(title,
description,
event_type,
source,
method,
reference,
date,
analyst,
bucket_list=None,
ticket=None):
"""
Add a new Event to CRITs.
:param title: Event title.
:type title: str
:param description: Event description.
:type description: str
:param event_type: Event type.
:type event_type: str
:param source: The source which provided this information.
:type source: str
:param method: THe method of acquiring this information.
:type method: str
:param reference: Reference to this data.
:type reference: str
:param date: Date of acquiring this data.
:type date: datetime.datetime
:param analyst: The user adding this Event.
:type analyst: str
:param bucket_list: The bucket(s) to associate with this Event.
:type: str
:param ticket: Ticket to associate with this event.
:type ticket: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if not source:
return {'success': False, 'message': "Missing source information."}
event = Event()
event.title = title
event.description = description
event.set_event_type(event_type)
s = create_embedded_source(name=source,
reference=reference,
method=method,
analyst=analyst,
date=date)
event.add_source(s)
if bucket_list:
event.add_bucket_list(bucket_list, analyst)
if ticket:
event.add_ticket(ticket, analyst)
try:
event.save(username=analyst)
# run event triage
event.reload()
run_triage(event, analyst)
message = (
'<div>Success! Click here to view the new event: <a href='
'"%s">%s</a></div>' %
(reverse('crits.events.views.view_event', args=[event.id]), title))
result = {
'success': True,
'message': message,
'id': str(event.id),
'object': event
}
except ValidationError, e:
result = {'success': False, 'message': e}
def handle_pcap_file(filename, data, source_name, user=None,
description=None, related_id=None, related_md5=None,
related_type=None, method='', reference='', tlp='',
relationship=None, bucket_list=None, ticket=None):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param reference: A reference to the source of this PCAP.
:type reference: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {
'success': False,
'message': 'No data object passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if ((related_id or related_md5) and not related_type):
status = {
'success': False,
'message': 'Must specify both related_type and related_id or related_md5.'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {
'success': False,
'message': 'Related object not found.'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
method=method,
reference=reference,
tlp=tlp,
analyst=user.username)
else:
return {"success":False,
"message": "User does not have permission to add object \
using source %s." % source_name}
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s, method=method, reference=reference)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if relationship:
relationship=RelationshipTypes.inverse(relationship=relationship)
else:
relationship = RelationshipTypes.RELATED_TO
pcap.add_relationship(related_obj,
relationship,
analyst=user,
get_rels=False)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(pcap, user)
status = {
'success': True,
'message': 'Uploaded pcap',
'md5': md5,
'id': str(pcap.id),
'object': pcap
}
return status
def add_new_domain(data,
request,
errors,
rowData=None,
is_validate_only=False,
cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: tuple
"""
username = request.user.username
result = False
retVal = {}
reference = data.get('domain_reference')
name = data.get('domain_source')
method = data.get('domain_method')
source = [
create_embedded_source(name,
reference=reference,
method=method,
analyst=username)
]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [
EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)
]
else:
campaign = []
(sdomain, fqdn) = get_domain(data['domain'])
if sdomain == "no_tld_found_error":
errors.append(u"Error: Invalid domain: " + data['domain'])
elif is_validate_only == False:
retVal = upsert_domain(sdomain,
fqdn,
source,
username,
campaign,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
ip_result = None
if retVal['success']:
new_domain = retVal['object']
add_ip = data.get('add_ip')
if add_ip:
ip = data.get('ip')
ip_type = data.get('ip_type')
if data.get('same_source'):
ip_source = data.get('domain_source')
ip_method = data.get('domain_method')
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(rel_item=new_ip,
rel_type='Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
new_ip.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail',
args=[fqdn])
if retVal['is_domain_new'] == True:
retVal['message'] = (
'Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, fqdn))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' %
(resp_url, fqdn))
retVal['message'] = message
retVal[form_consts.Status.
STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_obj
# If we have an IP object, add an indicator for that.
if ip_result and ip_result['success']:
obj = ip_result['object']
result = create_indicator_from_obj('Address - ipv4-addr',
'IP', obj.id, obj.ip,
username)
if result['success'] == False:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_obj('URI - Domain Name',
'Domain', new_domain.id,
sdomain, username)
if result['success'] == False:
errors.append(result['message'])
# If we have an FQDN (ie: it is not the same as sdomain)
# then add that also.
if fqdn != sdomain:
result = create_indicator_from_obj('URI - Domain Name',
'Domain', new_domain.id,
fqdn, username)
if result['success'] == False:
errors.append(result['message'])
result = True
elif 'message' in retVal: #database error? (!c_dom)
errors.append(
retVal['message']) #u"Unknown error: unable to add domain")
elif is_validate_only == True:
domain = data['domain']
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail',
args=[fqdn])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, fqdn))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
result_cache[domain.lower()] = True
# This block tries to add objects to the item
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(fqdn, cache)
for object_row_counter, objectData in enumerate(
objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(
objectData, "Domain", current_domain.id)
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
current_domain = None
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
(object_result, object_errors,
object_retVal) = validate_and_add_new_handler_object(
None,
objectDict,
request,
errors,
object_row_counter,
is_validate_only=is_validate_only,
cache=cache,
obj=current_domain)
if object_retVal.get('success') == False:
retVal['success'] = False
if object_retVal.get('message'):
errors.append(object_retVal['message'])
return result, errors, retVal
def add_new_backdoor(name, version=None, aliases=None, description=None,
source=None, source_method=None, source_reference=None,
campaign=None, confidence=None, user=None,
bucket_list=None, ticket=None):
"""
Add an Backdoor to CRITs.
:param name: The name of the backdoor.
:type name: str
:param version: Version of the backdoor.
:type version: str
:param aliases: Aliases for the backdoor.
:type aliases: list or str
:param description: Description of the backdoor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this backdoor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this backdoor.
:type user: str
:param bucket_list: Buckets to assign to this backdoor.
:type bucket_list: str
:param ticket: Ticket to assign to this backdoor.
:type ticket: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"id" (str),
"object" (if successful) :class:`crits.backdoors.backdoor.Backdoor`
"""
retVal = {'success': False, 'message': ''}
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=user)]
elif isinstance(source, EmbeddedSource):
source = [source]
if not source:
retVal['message'] = "Missing source information."
return retVal
# When creating a backdoor object we can potentially create multiple
# objects. If we are given a name but no version we will create an object
# with just the name (called the "family backdoor"). If given a name and a
# version we will create the family backdoor and the specific backdoor for
# that given version.
# In case we create more than one backdoor object, store the created ones
# in this list. The list is walked later on and attributes applied to each
# object.
objs = []
# First check if we have the family (name and no version).
family = Backdoor.objects(name=name, version='').first()
if not family:
# Family does not exist, new object. Details are handled later.
family = Backdoor()
family.name = name
family.version = ''
objs.append(family)
# Now check if we have the specific instance for this name + version.
backdoor = None
if version:
backdoor = Backdoor.objects(name=name, version=version).first()
if not backdoor:
# Backdoor does not exist, new object. Details are handled later.
backdoor = Backdoor()
backdoor.name = name
backdoor.version = version
objs.append(backdoor)
# At this point we have a family object and potentially a specific object.
# Add the common parameters to all objects in the list and save them.
for backdoor in objs:
for s in source:
backdoor.add_source(s)
# Don't overwrite existing description.
if description and backdoor.description == '':
backdoor.description = description.strip()
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=user)
campaign = [c]
if campaign:
for camp in campaign:
backdoor.add_campaign(camp)
if aliases:
if isinstance(aliases, basestring):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in backdoor.aliases:
backdoor.aliases.append(alias)
if bucket_list:
backdoor.add_bucket_list(bucket_list, user)
if ticket:
backdoor.add_ticket(ticket, user)
backdoor.save(username=user)
# run backdoor triage
backdoor.reload()
run_triage(backdoor, user)
# Because family objects are put in the list first we will always
# return a link to the most specific object created. If there is only
# one item in the list it will be the family object.
resp_url = reverse('crits.backdoors.views.backdoor_detail',
args=[backdoor.id])
retVal['message'] = 'Success: <a href="%s">%s</a>' % (resp_url,
backdoor.name)
retVal['object'] = backdoor
retVal['id'] = str(backdoor.id)
# If we have a family and specific object, attempt to relate the two.
if len(objs) == 2:
objs[0].add_relationship(objs[1], RelationshipTypes.RELATED_TO)
objs[0].save()
retVal['success'] = True
return retVal
def ip_add_update(ip_address, ip_type, source=None, source_method='',
source_reference='', source_tlp=None, campaign=None,
confidence='low', user=None, is_add_indicator=False,
indicator_reference='', bucket_list=None, ticket=None,
is_validate_only=False, cache={}, related_id=None,
related_type=None, relationship_type=None, description=''):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding/updating this IP.
:type user: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:param description: A description for this IP
:type description: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
if not source:
return {"success" : False, "message" : "Missing source information."}
source_name = source
(ip_address, error) = validate_and_normalize_ip(ip_address, ip_type)
if error:
return {"success": False, "message": error}
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if not ip_object.description:
ip_object.description = description or ''
elif ip_object.description != description:
ip_object.description += "\n" + (description or '')
if isinstance(source_name, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username)]
else:
return {"success":False,
"message": "User does not have permission to add object \
using source %s." % source}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user.username)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if bucket_list:
ip_object.add_bucket_list(bucket_list, user.username)
if ticket:
ip_object.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=user.username)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
result = handle_indicator_ind(ip_address,
source_name,
ip_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=source_method,
source_reference = indicator_reference,
source_tlp = source_tlp,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if related_obj and ip_object and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
ip_object.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
ip_object.save(username=user.username)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(ip_object, user)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def add_new_exploit(name, cve=None, description=None, source=None,
source_method=None, source_reference=None, campaign=None,
confidence=None, user=None, bucket_list=None,
ticket=None):
"""
Add an Exploit to CRITs.
:param name: The name of the exploit.
:type name: str
:param cve: CVE for the exploit.
:type cve: str
:param description: Description of the exploit.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this exploit.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this exploit.
:type user: str
:param bucket_list: Buckets to assign to this exploit.
:type bucket_list: str
:param ticket: Ticket to assign to this exploit.
:type ticket: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.exploits.exploit.Exploit`
"""
is_item_new = False
retVal = {}
exploit = Exploit.objects(name=name).first()
if not exploit:
exploit = Exploit()
exploit.name = name
if description:
exploit.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=user)]
elif isinstance(source, EmbeddedSource):
source = [source]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user)
campaign = [c]
if campaign:
for camp in campaign:
exploit.add_campaign(camp)
if source:
for s in source:
exploit.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
exploit.cve = cve.strip()
if bucket_list:
exploit.add_bucket_list(bucket_list, user)
if ticket:
exploit.add_ticket(ticket, user)
exploit.save(username=user)
# run exploit triage
if is_item_new:
exploit.reload()
run_triage(exploit, user)
resp_url = reverse('crits.exploits.views.exploit_detail', args=[exploit.id])
retVal['message'] = ('Success! Click here to view the new Exploit: '
'<a href="%s">%s</a>' % (resp_url, exploit.name))
retVal['success'] = True
retVal['object'] = exploit
retVal['id'] = str(exploit.id)
return retVal
