def get_md5_objects(oid, sources, md5_list=[], x=0):
obj_list = []
s = class_from_id('Sample', oid)
if not s:
return obj_list
if not source_match(s.source, sources):
return obj_list
md5_list.append(s.md5)
for o in s.obj:
if o.object_type in [ObjectTypes.DOMAIN, ObjectTypes.IPV4_ADDRESS, ObjectTypes.C2_URL] and source_match(o.source, sources):
obj_list.append(o.value)
for r in s.relationships:
if r.rel_type == 'Sample':
s2 = class_from_id('Sample', r.object_id)
if not s2:
continue
if not source_match(s2.source, sources):
continue
if s2.md5 not in md5_list and x < 1:
obj_list += get_md5_objects(r.object_id, sources, md5_list, x + 1)
return obj_list
def get_md5_objects(oid, sources, md5_list=[], x=0):
obj_list = []
s = class_from_id('Sample', oid)
if not s:
return obj_list
if not source_match(s.source, sources):
return obj_list
md5_list.append(s.md5)
for o in s.obj:
if o.name in ['Domain Name', 'ipv4-addr', 'URL'] and source_match(o.source, sources):
obj_list.append(o.value)
for r in s.relationships:
if r.rel_type == 'Sample':
s2 = class_from_id('Sample', r.object_id)
if not s2:
continue
if not source_match(s2.source, sources):
continue
if s2.md5 not in md5_list and x < 1:
obj_list += get_md5_objects(r.object_id, sources, md5_list, x + 1)
return obj_list
def get_sample_rels(rel, eid, sources):
s_list = []
for r in rel:
if r.rel_type == 'Sample':
s = class_from_id(r.rel_type, r.object_id)
if not s:
continue
if not source_match(s.source, sources):
continue
obj_list = get_md5_objects(r.object_id, sources)
backdoor = s.backdoor
if backdoor:
backdoor_name = s.backdoor.name
else:
backdoor_name = "None"
s_list.append({
'md5': s.md5,
'email_id': eid,
'mimetype': s.mimetype,
'filename': s.filename,
'backdoor': backdoor_name,
'objects': obj_list,
})
return s_list
def get_sample_rels(rel, eid, sources):
s_list = []
for r in rel:
if r.rel_type == 'Sample':
s = class_from_id(r.rel_type, r.object_id)
if not s:
continue
if not source_match(s.source, sources):
continue
obj_list = get_md5_objects(r.object_id, sources)
# Walk the relationships on this sample, see if it is related to
# a backdoor. Take the first backdoor that comes up, it may or
# may not be the versioned one.
backdoor_name = "None"
for sample_r in s.relationships:
if sample_r.rel_type == 'Backdoor':
backdoor = Backdoor.objects(id=sample_r.object_id).first()
if backdoor and source_match(backdoor.source, sources):
backdoor_name = backdoor.name
break
s_list.append({
'md5': s.md5,
'email_id': eid,
'mimetype': s.mimetype,
'filename': s.filename,
'backdoor': backdoor_name,
'objects': obj_list,
})
return s_list
def add_campaign_from_nodes(name, confidence, nodes, user):
result = { "success": False }
# Make sure Campaign exists
campaign_obj = Campaign.objects(name=name).first()
if not campaign_obj:
result["message"] = "Invalid campaign name."
return result
campaign = EmbeddedCampaign(name=name, confidence=confidence, analyst=user)
counter = 0
for node in nodes:
id_ = node.get('id', None)
type_ = node.get('type', None)
# Must have type and id, and type must not be Campaign
if not id_ or not type_ or type_.lower() == 'campaign':
continue
obj = class_from_id(type_, id_)
if not obj:
continue
obj.add_campaign(campaign)
obj.save()
counter += 1
result["message"] = "%s nodes processed" % counter
result["success"] = True
return result
def campaign_remove(ctype, oid, campaign, analyst):
"""
Remove Campaign attribution.
:param ctype: The top-level object type.
:type ctype: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param campaign: The Campaign to remove.
:type campaign: str
:param analyst: The user removing this attribution.
:type analyst: str
:returns: dict with key 'success' (boolean) and 'message' (str) if failed.
"""
# Verify the document exists.
crits_object = class_from_id(ctype, oid)
if not crits_object:
return {'success': False, 'message': 'Cannot find %s.' % ctype}
crits_object.remove_campaign(campaign)
try:
crits_object.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def refresh_services(request, crits_type, identifier):
"""
Refresh the Analysis tab with the latest information.
"""
response = {}
obj = class_from_id(crits_type, identifier)
if not obj:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), mimetype="application/json")
relationship = {'type': crits_type,
'value': identifier}
subscription = {'type': crits_type,
'id': identifier}
service_list = get_supported_services(crits_type)
response['success'] = True
response['html'] = render_to_string("services_analysis_listing.html",
{'relationship': relationship,
'subscription': subscription,
'item': obj,
'crits_type': crits_type,
'identifier': identifier,
'service_list': service_list},
RequestContext(request))
return HttpResponse(json.dumps(response), mimetype="application/json")
def delete_screenshot_from_object(obj, oid, sid, analyst):
"""
Remove a screenshot from a top-level object.
:param obj: The type of top-level object to work with.
:type obj: str
:param oid: The ObjectId of the top-level object to work with.
:type oid: str
:param sid: The ObjectId of the screenshot to remove.
:type sid: str
:param analyst: The user removing the screenshot.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str).
"""
result = {'success': False}
klass = class_from_id(obj, oid)
if not klass:
result['message'] = "Could not find Object to delete screenshot from."
return result
clean = [s for s in klass.screenshots if s != sid]
klass.screenshots = clean
try:
klass.save(username=analyst)
result['success'] = True
return result
except Exception, e:
result['message'] = str(e)
return result
def delete_all_relationships(left_class=None, left_type=None,
left_id=None, analyst=None):
"""
Delete all relationships for this top-level object.
:param left_class: The top-level object to delete relationships for.
:type left_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param left_type: The type of the top-level object.
:type left_type: str
:param left_id: The ObjectId of the top-level object.
:type left_id: str
:param analyst: The user deleting these relationships.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if not left_class:
if left_type and left_id:
left_class = class_from_id(left_type, left_id)
if not left_class:
return {'success': False,
'message': "Unable to get object."}
else:
return {'success': False,
'message': "Need a valid left type and id"}
return left_class.delete_all_relationships()
def get_taxii_result(request, crits_type, crits_id, preview):
"""
Create the STIX document representing the given CRITs object.
If preview, download the STIX file for user to peruse, else
wrap in a TAXII message and send to TAXII server.
:param request The request object
:param crits_type The type of the crits object that will be converted
:param crits_id The ID of the crits object that will be converted
:param preview Boolean flag indicating if this is a preview generation or message send req
"""
obj = class_from_id(crits_type, crits_id)
if not obj:
ret = {'success': False, 'reason': "Could not locate object in the database."}
return HttpResponse(json.dumps(ret), mimetype="application/json")
# did user accept responsibility for potential releasability updates?
confirm_rel = True if "updates_confirmed" in request.POST else False
form = forms.TAXIIForm(request.user.username, obj, request.GET if preview else request.POST)
if form.is_valid(): # is_valid seems to ensure that multiselect data was all in original form
rcpts = form.cleaned_data.get('rcpts', [])
relation_choices = form.get_chosen_relations()
data = handlers.run_taxii_service(request.user.username, obj, rcpts, preview, relation_choices, confirm_rel)
if preview and data and 'preview' in data: # if doing preview and data available, download as file
resp = HttpResponse(data['preview'], content_type="application/xml")
resp['Content-Disposition'] = 'attachment; filename="STIX_preview.xml"'
return resp
else: # else show success/error message that has been generated
return HttpResponse(json.dumps(data), mimetype="application/json")
else: # form doesn't validate
data = {'success': False, 'reason': "Invalid options provided. Please fix and try again."}
return HttpResponse(json.dumps(data), mimetype="application/json")
def update_object_value(type_, oid, object_type, name, value, new_value,
analyst):
"""
Update an object value.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the object to update.
:type object_type: str
:param name: The name of the object to update.
:type name: str
:param value: The value of the object to update.
:type value: str
:param new_value: The new value to use.
:type new_value: str
:param analyst: The user removing this object.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
obj = class_from_id(type_, oid)
if not obj:
return {'success': False,
'message': "Could not find item to update object."}
try:
obj.update_object_value(object_type,
name,
value,
new_value)
obj.save(username=analyst)
return {'success': True, 'message': 'Object value updated successfully.'}
except ValidationError, e:
return {'success': False, 'message': e}
def location_remove(id_, type_, location_name, location_type, date, user):
"""
Remove location attribution.
:param id_: The ObjectId of the TLO.
:type id_: str
:param type_: The type of TLO.
:type type_: str
:param location_name: The location to remove.
:type location_name: str
:param location_type: The location type to remove.
:type location_type: str
:param date: The location date to remove.
:type date: str
:param user: The user removing this attribution.
:type user: str
:returns: dict with key 'success' (boolean) and 'message' (str) if failed.
"""
# Verify the document exists.
crits_object = class_from_id(type_, id_)
if not crits_object:
return {'success': False, 'message': 'Cannot find %s.' % type_}
crits_object.remove_location(location_name, location_type, date)
try:
crits_object.save(username=user)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def location_add(id_, type_, location_type, location_name, user,
description=None, latitude=None, longitude=None):
"""
Add a Location to a top-level object.
:param id_: The ObjectId of the TLO to add this location to.
:type id_: str
:param type_: The TLO type.
:type type_: str
:param location_type: The type of location based on origin or destination.
:type location_type: str
:param location: The location.
:type location: str
:param user: The user attributing this Location.
:type user: str
:param description: Description of this attribution.
:type description: str
:param latitude: The latitude of the location.
:type latitude: str
:param longitude: The longitude of the location.
:type longitude: str
:returns: dict with keys:
'success' (boolean),
'html' (str) if successful,
'message' (str).
"""
if id_ and type_:
# Verify the document exists.
obj = class_from_id(type_, id_)
if not obj:
return {'success': False, 'message': 'Cannot find %s.' % type_}
else:
return {'success': False,
'message': 'Object type and ID must be provided.'}
# Create the embedded location.
location = EmbeddedLocation(
location_type = location_type,
location = location_name,
description = description,
latitude = latitude,
longitude = longitude,
analyst = user.username
)
location.date = location.date.replace(microsecond=0)
result = obj.add_location(location)
if result['success']:
try:
obj.save(username=user)
html = obj.format_location(location, user)
return {'success': True,
'html': html,
'message': result['message']}
except ValidationError, e:
return {'success': False,
'message': "Invalid value: %s" % e}
def create_indicator_from_obj(ind_type, obj_type, id_, value, analyst):
"""
Add indicators from CRITs object.
:param ind_type: The indicator type to add.
:type ind_type: str
:param obj_type: The CRITs type of the parent object.
:type obj_type: str
:param id_: The ObjectId of the parent object.
:type id_: str
:param value: The value of the indicator to add.
:type value: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
obj = class_from_id(obj_type, id_)
if not obj:
return {'success': False, 'message': 'Could not find object.'}
source = obj.source
bucket_list = obj.bucket_list
campaign = None
campaign_confidence = None
if len(obj.campaign) > 0:
campaign = obj.campaign[0].name
campaign_confidence = obj.campaign[0].confidence
result = handle_indicator_ind(value, source, reference=None, ctype=ind_type,
analyst=analyst,
add_domain=True,
add_relationship=True,
campaign=campaign,
campaign_confidence=campaign_confidence,
bucket_list=bucket_list)
if result['success']:
ind = Indicator.objects(id=result['objectid']).first()
if ind:
obj.add_relationship(rel_item=ind,
rel_type="Related_To",
analyst=analyst)
obj.save(username=analyst)
for rel in obj.relationships:
if rel.rel_type == "Event":
ind.add_relationship(rel_id=rel.object_id,
type_=rel.rel_type,
rel_type="Related_To",
analyst=analyst)
ind.save(username=analyst)
obj.reload()
rels = obj.sort_relationships("%s" % analyst, meta=True)
return {'success': True, 'message': rels, 'value': id_}
else:
return {'success': False, 'message': result['message']}
def run(self, argv):
parser = OptionParser()
parser.add_option('-d', '--domain', dest='domain', help='domain')
parser.add_option('-l', '--domain_list', dest='domain_list', help='domain list')
parser.add_option('-i', '--id_list', dest='id_list', help='id list')
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
default=False,
help='Verbose mode')
parser.add_option('-T', '--type', dest='type_', default='Domain',
help='CRITs type query for (default: Domain)')
parser.add_option('--delete', dest='delete', action='store_true',
default=False,
help='Delete Domains')
parser.add_option('--csv', dest='csv', action='store_true',
default=False,
help='Treat file as CSV')
(opts, args) = parser.parse_args(argv)
domain_list = []
if opts.domain_list:
with open(opts.domain_list) as infile:
for line in infile:
if opts.csv:
domain_list = [x.strip() for x in line.split(',')]
else:
domain_list.append(line.strip())
elif opts.domain:
domain_list = opts.domain.split(',')
if opts.verbose:
self.print_delete_objects(domain_list)
error_list = []
obj_list = []
for domain in domain_list:
#domain_obj = Domain.objects(domain_iexact=domain).first()
obj = class_from_value(opts.type_, domain)
if not obj:
error_list.append(domain)
else:
obj_list.append(obj)
if opts.verbose:
self.print_found_objects(obj_list, error_list)
if opts.id_list:
with open(opts.id_list) as infile:
for line in infile:
result = json.loads(line.strip())
obj = class_from_id(opts.type_, result['object_id'])
obj_list.append(obj)
if opts.delete:
self.delete_domains(obj_list, opts.type_, 0.5)
else:
self.run_analysis_cleanup(obj_list, opts.type_, 0.5)
print("Done!")
def campaign_add(campaign_name, confidence, description, related,
analyst, ctype=None, oid=None, obj=None, update=True):
"""
Attribute a Campaign to a top-level object.
:param campaign_name: The Campaign to attribute.
:type campaign_name: str
:param confidence: The confidence level of this attribution (low, medium, high)
:type confidence: str
:param description: Description of this attribution.
:type description: str
:param related: Should this attribution propagate to related top-level objects.
:type related: boolean
:param analyst: The user attributing this Campaign.
:type analyst: str
:param ctype: The top-level object type.
:type ctype: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param obj: The top-level object instantiated class.
:type obj: Instantiated class object
:param update: If True, allow merge with pre-existing campaigns
: If False, do not change any pre-existing campaigns
:type update: boolean
:returns: dict with keys:
'success' (boolean),
'html' (str) if successful,
'message' (str).
"""
if not obj:
if ctype and oid:
# Verify the document exists.
obj = class_from_id(ctype, oid)
if not obj:
return {'success': False, 'message': 'Cannot find %s.' % ctype}
else:
return {'success': False, 'message': 'Object type and ID, or object instance, must be provided.'}
# Create the embedded campaign.
campaign = EmbeddedCampaign(name=campaign_name, confidence=confidence, description=description, analyst=analyst)
result = obj.add_campaign(campaign, update=update)
if result['success']:
if related:
campaign_addto_related(obj, campaign, analyst)
try:
obj.save(username=analyst)
html = obj.format_campaign(campaign, analyst)
return {'success': True, 'html': html, 'message': result['message']}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def delete_analysis(crits_type, identifier, task_id, analyst):
"""
Delete analysis results.
"""
obj = class_from_id(crits_type, identifier)
if obj:
c = 0
for a in obj.analysis:
if str(a.analysis_id) == task_id:
del obj.analysis[c]
c += 1
obj.save(username=analyst)
def delete_object(type_, oid, object_type, name, value, analyst, get_objects=True):
"""
Delete an object.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the object to remove.
:type object_type: str
:param name: The name of the object to remove.
:type name: str
:param value: The value of the object to remove.
:type value: str
:param analyst: The user removing this object.
:type analyst: str
:param get_objects: Return the list of objects.
:type get_objects: bool
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list)
"""
obj = class_from_id(type_, oid)
if not obj:
return {'success': False,
'message': "Could not find item to remove object from."}
try:
cur_len = len(obj.obj)
obj.remove_object(object_type,
name,
value)
obj.save(username=analyst)
new_len = len(obj.obj)
result = {}
if new_len < cur_len:
result['success'] = True
result['message'] = "Object removed successfully!"
else:
result['success'] = False
result['message'] = "Could not find object to remove!"
if (get_objects):
result['objects'] = obj.sort_objects()
return result
except ValidationError, e:
return {'success': False,
'message': e}
def get_taxii_config_form(request, crits_type, crits_id):
if request.method == "GET":
obj = class_from_id(crits_type, crits_id)
if not obj:
ret = {'success': False, 'reason': "Could not locate object in the database."}
return HttpResponse(json.dumps(ret), mimetype="application/json")
tform = forms.TAXIIForm(request.user.username, obj)
taxii_form = {'form' : render_to_string("_taxii_form_template.html", {'form' : tform})}
return HttpResponse(json.dumps(taxii_form), mimetype="application/json")
else:
return render_to_response('error.html',
{'error': "Must be AJAX."},
RequestContext(request))
def campaign_edit(ctype, oid, campaign_name, confidence,
description, date, related, analyst):
"""
Edit an attributed Campaign for a top-level object.
:param ctype: The top-level object type.
:type ctype: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param campaign_name: The Campaign to attribute.
:type campaign_name: str
:param confidence: The confidence level of this attribution (low, medium, high)
:type confidence: str
:param description: Description of this attribution.
:type description: str
:param date: The date of attribution.
:type date: :class:`datetime.datetime`
:param related: Should this attribution propagate to related top-level objects.
:type related: boolean
:param analyst: The user editing this attribution.
:type analyst: str
:returns: dict with keys:
'success' (boolean),
'html' (str) if successful,
'message' (str) if failed.
"""
# Verify the document exists.
crits_object = class_from_id(ctype, oid)
if not crits_object:
return {'success': False, 'message': 'Cannot find %s.' % ctype}
# Create the embedded campaign.
campaign = EmbeddedCampaign(name=campaign_name, confidence=confidence,
description=description, analyst=analyst,
date=date)
crits_object.edit_campaign(campaign_item=campaign)
if related:
campaign_addto_related(crits_object, campaign, analyst)
try:
crits_object.save(username=analyst)
html = crits_object.format_campaign(campaign, analyst)
return {'success': True, 'html': html}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def run(self, obj, config):
self.config = config
self.obj = obj
self._debug("pdf2txt started\n")
pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext")
# The _write_to_file() context manager will delete this file at the
# end of the "with" block.
with self._write_to_file() as tmp_file:
(working_dir, filename) = os.path.split(tmp_file)
args = [pdf2txt_path, filename, "-"]
# pdftotext does not generate a lot of output, so we should not have to
# worry about this hanging because the buffer is full
proc = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=working_dir)
# Note that we are redirecting STDERR to STDOUT, so we can ignore
# the second element of the tuple returned by communicate().
output = proc.communicate()[0]
self._debug(output)
if proc.returncode:
msg = ("pdftotext could not process the file.")
self._warning(msg)
return
raw_hash = md5(output).hexdigest()
res = handle_raw_data_file(output, self.obj.source, self.current_task.username,
title="pdftotext", data_type='text',
tool_name='pdftotext', tool_version='0.1', tool_details='http://poppler.freedesktop.org',
method=self.name,
copy_rels=True)
raw_obj = class_from_id("RawData", res["_id"])
self._warning("obj.id: %s, raw_id:%s, suc: %s" % (str(obj.id), str(raw_obj.id), repr(res['success']) ) )
# update relationship if a related top-level object is supplied
rel_type = RelationshipTypes.RELATED_TO
if obj.id != raw_obj.id: #don't form relationship to itself
resy = obj.add_relationship(rel_item=raw_obj,
rel_type=rel_type,
rel_date=datetime.now(),
analyst=self.current_task.username)
obj.save(username=self.current_task.username)
raw_obj.save(username=self.current_task.username)
self._warning("resy: %s" % (str(resy)) )
self._add_result("rawdata_added", raw_hash, {'md5': raw_hash})
return
def get_screenshots_for_id(type_, _id, analyst, buckets=False):
"""
Get screenshots for a top-level object.
:param type_: The class type.
:type type_: str
:param _id: The ObjectId to lookup.
:type _id: str
:param analyst: The user looking up the screenshots.
:type analyst: str
:param buckets: Use buckets as tag lookups for screenshots.
:type buckets: boolean
:returns: list
"""
result = {'success': False}
sources = user_sources(analyst)
obj = class_from_id(type_, _id)
if not obj:
result['message'] = "No valid top-level object found."
return result
screenshots = Screenshot.objects(id__in=obj.screenshots,
source__name__in=sources)
bucket_shots = Screenshot.objects(tags__in=obj.bucket_list,
source__name__in=sources)
final_shots = []
for s in screenshots:
if s.screenshot and s.thumb and s not in final_shots:
final_shots.append(s)
for b in bucket_shots:
if b not in final_shots:
# since .bucket isn't supported, this will show up in the template
# under unsupported_attrs, which is ok.
b.bucket = True
final_shots.append(b)
result['success'] = True
result['screenshots'] = final_shots
return result
def location_edit(type_, id_, location_name, location_type, date, user,
description=None, latitude=None, longitude=None):
"""
Update a location.
:param type_: Type of TLO.
:type type_: str
:param id_: The ObjectId of the TLO.
:type id_: str
:param location_name: The name of the location to change.
:type location_name: str
:param location_type: The type of the location to change.
:type location_type: str
:param date: The location date to edit.
:type date: str
:param user: The user setting the new description.
:type user: str
:param description: The new description.
:type description: str
:param latitude: The new latitude.
:type latitude: str
:param longitude: The new longitude.
:type longitude: str
:returns: dict with key 'success' (boolean) and 'message' (str) if failed.
"""
crits_object = class_from_id(type_, id_)
if not crits_object:
return {'success': False, 'message': 'Cannot find %s.' % type_}
crits_object.edit_location(location_name,
location_type,
date,
description=description,
latitude=latitude,
longitude=longitude)
try:
crits_object.save(username=user)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def get_relationships(obj=None, type_=None, id_=None, analyst=None):
"""
Get relationships for a top-level object.
:param obj: The top-level object to get relationships for.
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param type_: The top-level object type to get relationships for.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param analyst: The user requesting the relationships.
:type analyst: str
:returns: dict
"""
if obj:
return obj.sort_relationships("%s" % analyst, meta=True)
elif type_ and id_:
obj = class_from_id(type_, id_)
return obj.sort_relationships("%s" % analyst, meta=True)
else:
return {}
def add_new_handler_object(data, rowData, request, is_validate_only=False,
is_sort_relationships=False, cache={}, obj=None):
"""
Add an object to the database.
:param data: The data for the object.
:type data: dict
:param rowData: Data from the row if using mass object upload.
:type rowData: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param is_validate_only: Only validate.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:returns: tuple (<result>, <retVal>)
"""
result = False
retVal = {}
username = request.user.username
if data:
object_type = data.get('object_type')
value = data.get('value')
source = data.get('source')
method = data.get('method')
reference = data.get('reference')
otype = data.get('otype')
oid = data.get('oid')
add_indicator = data.get('add_indicator')
elif rowData:
object_type = rowData.get(form_consts.Object.OBJECT_TYPE)
value = rowData.get(form_consts.Object.VALUE)
source = rowData.get(form_consts.Object.SOURCE)
method = rowData.get(form_consts.Object.METHOD)
reference = rowData.get(form_consts.Object.REFERENCE)
otype = rowData.get(form_consts.Object.PARENT_OBJECT_TYPE)
oid = rowData.get(form_consts.Object.PARENT_OBJECT_ID)
add_indicator = rowData.get(form_consts.Object.ADD_INDICATOR)
is_validate_locally = False
analyst = "%s" % username
# Default the user source to the user's organization if not specified
if not source:
source = cache.get('object_user_source')
if not source:
source = get_user_organization(analyst)
cache['object_user_source'] = source
if (otype == "" or otype == None) or (oid == "" or oid == None):
is_validate_locally = True
# TODO file_
object_result = add_object(
otype, oid, object_type, source, method, reference, analyst,
value=value, file_=None, add_indicator=add_indicator, get_objects=False,
tlo=obj, is_validate_only=is_validate_only,
is_sort_relationships=is_sort_relationships,
is_validate_locally=is_validate_locally, cache=cache
)
if object_result['success']:
result = True
if 'message' in object_result:
retVal['message'] = object_result['message']
if is_validate_only == False:
if obj == None:
obj = class_from_id(otype, oid)
if obj:
retVal['secondary'] = {'type': otype, 'id': oid}
if object_result.get('relationships'):
retVal['secondary']['relationships'] = object_result.get('relationships')
else:
retVal['message'] = object_result['message']
return result, retVal
def forge_relationship(type_=None, id_=None,
class_=None, right_type=None,
right_id=None, right_class=None,
rel_type=None, rel_date=None,
user=None, rel_reason="",
rel_confidence='unknown', get_rels=False, **kwargs):
"""
Forge a relationship between two top-level objects.
:param type_: The type of first top-level object to relate to.
:type type_: str
:param id_: The ObjectId of the first top-level object.
:type id_: str
:param class_: The first top-level object to relate to.
:type class_: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param right_type: The type of second top-level object to relate to.
:type right_type: str
:param right_id: The ObjectId of the second top-level object.
:type right_id: str
:param right_class: The second top-level object to relate to.
:type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param rel_type: The type of relationship.
:type rel_type: str
:param rel_date: The date this relationship applies.
:type rel_date: datetime.datetime
:param user: The user forging this relationship.
:type user: str
:param rel_reason: The reason for the relationship.
:type rel_reason: str
:param rel_confidence: The confidence of the relationship.
:type rel_confidence: str
:param get_rels: Return the relationships after forging.
:type get_rels: boolean
:returns: dict with keys:
"success" (boolean)
"message" (str if fail, EmbeddedObject if success)
"relationships" (dict)
"""
if rel_date == 'None':
rel_date = None
elif isinstance(rel_date, basestring) and rel_date != '':
rel_date = parse(rel_date, fuzzy=True)
elif not isinstance(rel_date, datetime.datetime):
rel_date = None
if not class_:
if type_ and id_:
class_ = class_from_id(type_, id_)
if not class_:
return {'success': False, 'message': "Failed to get left TLO"}
if not right_class:
if right_type and right_id:
right_class = class_from_id(right_type, right_id)
if not right_class:
return {'success': False, 'message': "Failed to get right TLO"}
try:
# forge relationship
results = class_.add_relationship(right_class, rel_type, rel_date,
user, rel_confidence, rel_reason)
except Exception as e:
return {'success': False, 'message': e}
if results['success']:
class_.update(add_to_set__relationships=results['message'])
if get_rels:
results['relationships'] = class_.sort_relationships("%s" % user,
meta=True)
return results
def add_new_actor(name, aliases=None, description=None, source=None,
source_method='', source_reference='', source_tlp=None,
campaign=None, confidence=None, user=None,
bucket_list=None, ticket=None, related_id=None,
related_type=None, relationship_type=None):
"""
Add an Actor to CRITs.
:param name: The name of the Actor.
:type name: str
:param aliases: Aliases for the actor.
:type aliases: list or str
:param description: Description of the actor.
:type description: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param source_tlp: The TLP for this Actor.
:type source_tlp: str
:param campaign: A campaign to attribute to this actor.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding this actor.
:type user: :class:`crits.core.user.CRITsUser`
:param bucket_list: Buckets to assign to this actor.
:type bucket_list: str
:param ticket: Ticket to assign to this actor.
:type ticket: str
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.actors.actor.Actor`
"""
username = user.username
is_item_new = False
retVal = {}
actor = Actor.objects(name=name).first()
if not actor:
actor = Actor()
actor.name = name
if description:
actor.description = description.strip()
is_item_new = True
if isinstance(source, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=username)]
else:
return {"success": False,
"message": "User does not have permission to add objects \
using source %s." % str(source)}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
if campaign:
for camp in campaign:
actor.add_campaign(camp)
if source:
for s in source:
actor.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if not isinstance(aliases, list):
aliases = aliases.split(',')
for alias in aliases:
alias = alias.strip()
if alias not in actor.aliases:
actor.aliases.append(alias)
if bucket_list:
actor.add_bucket_list(bucket_list, username)
if ticket:
actor.add_ticket(ticket, username)
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
actor.save(username=username)
if related_obj and actor:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
actor.add_relationship(related_obj,
relationship_type,
analyst=username,
get_rels=False)
actor.save(username=username)
actor.reload()
# run actor triage
if is_item_new:
actor.reload()
run_triage(actor, user)
resp_url = reverse('crits-actors-views-actor_detail', args=[actor.id])
retVal['message'] = ('Success! Click here to view the new Actor: '
'<a href="%s">%s</a>' % (resp_url, actor.name))
retVal['success'] = True
retVal['object'] = actor
retVal['id'] = str(actor.id)
return retVal
def delete_relationship(left_class=None, right_class=None,
left_type=None, left_id=None,
right_type=None, right_id=None,
rel_type=None, rel_date=None,
analyst=None, get_rels=True):
"""
Delete a relationship between two top-level objects.
:param left_class: The first top-level object.
:type left_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param right_class: The second top-level object.
:type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param left_type: The type of first top-level object.
:type left_type: str
:param left_id: The ObjectId of the first top-level object.
:type left_id: str
:param right_type: The type of second top-level object.
:type right_type: str
:param right_id: The ObjectId of the second top-level object.
:type right_id: str
:param rel_type: The type of relationship.
:type rel_type: str
:param rel_date: The date this relationship applies.
:type rel_date: datetime.datetime
:param analyst: The user deleting this relationship.
:type analyst: str
:param get_rels: Return the relationships after forging.
:type get_rels: boolean
:returns: dict with keys "success" (boolean) and "message" (str if
failed, dict if successful)
"""
if rel_date is None or rel_date == 'None':
rel_date = None
elif isinstance(rel_date, basestring) and rel_date != '':
rel_date = parse(rel_date, fuzzy=True)
elif not isinstance(rel_date, datetime.datetime):
rel_date = None
if not left_class:
if left_type and left_id:
left_class = class_from_id(left_type, left_id)
if not left_class:
return {'success': False,
'message': "Unable to get object."}
else:
return {'success': False,
'message': "Need a valid left type and id"}
# delete relationship
if right_class:
results = left_class.delete_relationship(rel_item=right_class,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst)
else:
if right_type and right_id:
results = left_class.delete_relationship(type_=right_type,
rel_id=right_id,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst)
else:
return {'success': False,
'message': "Need a valid right type and id"}
if results['success']:
left_class.save(username=analyst)
if get_rels:
results['relationships'] = left_class.sort_relationships("%s" % analyst, meta=True)
return results
def to_stix(obj, items_to_convert=[], loaded=False, bin_fmt="raw", ref_id=None):
"""
Converts a CRITs object to a STIX document.
The resulting document includes standardized representations
of all related objects noted within items_to_convert.
:param items_to_convert: The list of items to convert to STIX/CybOX
:type items_to_convert: Either a list of CRITs objects OR
a list of {'_type': CRITS_TYPE, '_id': CRITS_ID} dicts
:param loaded: Set to True if you've passed a list of CRITs objects as
the value for items_to_convert, else leave False.
:type loaded: bool
:param bin_fmt: Specifies the format for Sample data encoding.
Options: None (don't include binary data in STIX output),
"raw" (include binary data as is),
"base64" (base64 encode binary data)
:returns: A dict indicating which items mapped to STIX indicators, ['stix_indicators']
which items mapped to STIX observables, ['stix_observables']
which items are included in the resulting STIX doc, ['final_objects']
and the STIX doc itself ['stix_obj'].
"""
from cybox.common import Time, ToolInformationList, ToolInformation
from stix.common import StructuredText, InformationSource
from stix.core import STIXPackage, STIXHeader
from stix.common.identity import Identity
import stix.common.kill_chains.lmco as lmco
# These lists are used to determine which CRITs objects
# go in which part of the STIX document.
ind_list = ['Indicator']
obs_list = ['Domain',
'Email',
'IP',
'Sample']
camp_list = ['Campaign']
actor_list = ['Actor']
# Store message
stix_msg = {
'stix_incidents': [],
'stix_indicators': [],
'stix_observables': [],
'stix_actors': [],
'final_objects': []
}
if not loaded: # if we have a list of object metadata, load it before processing
items_to_convert = [class_from_id(item['_type'], item['_id'])
for item in items_to_convert]
# add self to the list of items to STIXify
if obj not in items_to_convert:
items_to_convert.append(obj)
# add any email attachments
attachments = []
for obj in items_to_convert:
if obj._meta['crits_type'] == 'Email':
for rel in obj.relationships:
if rel.relationship == RelationshipTypes.CONTAINS:
atch = class_from_id('Sample', rel.object_id)
if atch not in items_to_convert:
attachments.append(atch)
items_to_convert.extend(attachments)
# grab ObjectId of items
refObjs = {key.id: 0 for key in items_to_convert}
relationships = {}
stix = []
stx = False
tlp = None
from stix.indicator import Indicator as S_Ind
for obj in items_to_convert:
obj_type = obj._meta['crits_type']
if obj_type == class_from_type('Event')._meta['crits_type']:
stx, release = to_stix_incident(obj)
stix_msg['stix_incidents'].append(stx)
elif obj_type in ind_list: # convert to STIX indicators
stx, releas = to_stix_indicator(obj)
stix_msg['stix_indicators'].append(stx)
refObjs[obj.id] = S_Ind(idref=stx.id_)
elif obj_type in camp_list:
camp = to_stix_campaign(obj, False)
comm = to_stix_comments(obj)
tlp = to_stix_tlp(obj)
rel = to_stix_relationship(obj)
sight = to_stix_sightings(obj)
kill = to_stix_kill_chains(obj)
ttp = to_stix_ttps(obj)
rfi = to_stix_rfi(obj)
ind = S_Ind()
ind.title = "MARTI Campaign"
ind.sightings.append(sight)
for each in camp:
ind.add_related_campaign(each)
for each in comm:
ind.add_related_indicator(each)
for each in rel:
ind.add_related_indicator(each)
for each in rfi:
ind.add_related_indicator(each)
for each in kill:
ind.add_kill_chain_phase(each)
for each in ttp:
ind.add_indicated_ttp(each)
ind.producer = to_stix_information_source(obj)
ind.short_descriptions = obj.sectors
ind.descriptions = obj.aliases
stx = ind
stix_msg['stix_indicators'].append(stx)
refObjs[obj.id] = S_Ind(idref=stx.id_)
elif obj_type in obs_list: # convert to CybOX observable
camp = to_stix_campaign(obj)
comm = to_stix_comments(obj)
rel = to_stix_relationship(obj)
sight = to_stix_sightings(obj)
kill = to_stix_kill_chains(obj)
tlp = to_stix_tlp(obj)
rfi = to_stix_rfi(obj)
if obj_type == class_from_type('Sample')._meta['crits_type']:
stx, releas = to_cybox_observable(obj, bin_fmt=bin_fmt)
else:
stx, releas = to_cybox_observable(obj)
# wrap in stix Indicator
ind = S_Ind()
for ob in stx:
ind.add_observable(ob)
ind.sightings.append(sight)
for each in camp:
ind.add_related_campaign(each)
for each in comm:
ind.add_related_indicator(each)
for each in rel:
ind.add_related_indicator(each)
for each in rfi:
ind.add_related_indicator(each)
for each in kill:
ind.add_kill_chain_phase(each)
ind.title = "CRITs %s Top-Level Object" % obj_type
ind.description = ("This is simply a CRITs %s top-level "
"object, not actually an Indicator. "
"The Observable is wrapped in an Indicator"
" to facilitate documentation of the "
"relationship." % obj_type)
ind.confidence = 'None'
ind.producer = to_stix_information_source(obj)
ind.short_descriptions = obj.sectors
stx = ind
stix_msg['stix_indicators'].append(stx)
refObjs[obj.id] = S_Ind(idref=stx.id_)
elif obj_type in actor_list: # convert to STIX actor
stx, releas = to_stix_actor(obj)
stix_msg['stix_actors'].append(stx)
# get relationships from CRITs objects
for rel in obj.relationships:
if rel.object_id in refObjs:
relationships.setdefault(stx.id_, {})
relationships[stx.id_][rel.object_id] = (rel.relationship,
rel.rel_confidence.capitalize(),
rel.rel_type)
stix_msg['final_objects'].append(obj)
if stx:
stix.append(stx)
# set relationships on STIX objects
for stix_obj in stix:
for rel in relationships.get(stix_obj.id_, {}):
if isinstance(refObjs.get(rel), S_Ind): # if is STIX Indicator
stix_obj.related_indicators.append(refObjs[rel])
rel_meta = relationships.get(stix_obj.id_)[rel]
stix_obj.related_indicators[-1].relationship = rel_meta[0]
stix_obj.related_indicators[-1].confidence = rel_meta[1]
# Add any Email Attachments to CybOX EmailMessage Objects
if isinstance(stix_obj, S_Ind):
if 'EmailMessage' in stix_obj.observable.object_.id_:
if rel_meta[0] == 'Contains' and rel_meta[2] == 'Sample':
email = stix_obj.observable.object_.properties
email.attachments.append(refObjs[rel].idref)
tool_list = ToolInformationList()
tool = ToolInformation("CRITs", "MITRE")
tool.version = settings.CRITS_VERSION
tool_list.append(tool)
i_s = InformationSource(
time=Time(produced_time= datetime.now()),
identity=Identity(name=settings.COMPANY_NAME),
tools=tool_list)
if obj._meta['crits_type'] == "Event":
stix_desc = obj.description()
stix_int = obj.event_type()
stix_title = obj.title()
else:
stix_desc = "STIX from %s" % settings.COMPANY_NAME
stix_int = "Collective Threat Intelligence"
stix_title = "Threat Intelligence Sharing"
header = STIXHeader(information_source=i_s,
description=StructuredText(value=stix_desc),
package_intents=[stix_int],
title=stix_title,
handling=tlp)
if not ref_id:
ref_id = uuid.uuid4()
stix_msg['stix_obj'] = STIXPackage(incidents=stix_msg['stix_incidents'],
indicators=stix_msg['stix_indicators'],
threat_actors=stix_msg['stix_actors'],
stix_header=header,
campaigns=camp,
id_=ref_id)
#print stix_msg['stix_obj'].to_xml()
return stix_msg
def create_indicator_from_object(rel_type, rel_id, ind_type, value, analyst,
request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False,
'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty value field"}
elif ind_type == None or ind_type.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty type field"}
else:
create_indicator_result = {}
ind_tlist = ind_type.split(" - ")
if ind_tlist[0] == ind_tlist[1]:
ind_type = ind_tlist[0]
from crits.indicators.handlers import handle_indicator_ind
if hasattr(me, 'source'):
create_indicator_result = handle_indicator_ind(value,
me.source,
'',
ind_type,
analyst=analyst,
add_domain=True)
else:
# In case the top level item doesn't have sources (such as campaign)...
# then just default to the user's organization
create_indicator_result = handle_indicator_ind(value,
get_user_organization(analyst),
'',
ind_type,
analyst=analyst,
add_domain=True)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return result
indicator = Indicator.objects(ind_type=ind_type,
value=value).first()
if not indicator:
result = {'success': False,
'message': "Could not create indicator"}
else:
results = me.add_relationship(rel_item=indicator,
rel_type="Related_To",
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
indicator.save(username=analyst)
relationship= {'type': rel_type, 'value': rel_id}
message = render_to_string('relationships_listing_widget.html',
{'relationship': relationship,
'nohide': True,
'relationships': results['message']},
RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False,
'message': message}
return result
def add_campaign(name,
description,
aliases,
analyst,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add a Campaign.
:param name: The name of the new Campaign.
:type name: str
:param description: Description of the new Campaign.
:type description: str
:param aliases: Aliases for the new Campaign.
:type aliases: str (comma separated) or list.
:param analyst: The user adding the Campaign.
:type analyst: str
:param bucket_list: Buckets to add to this Campaign.
:type bucket_list: str (comma separated) or list.
:param ticket: Ticket(s) to add to this Campaign.
:type ticket: str (comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with key 'success' (boolean) and 'message' (str).
"""
# Verify the Campaign does not exist.
campaign = Campaign.objects(name=name).first()
if campaign:
return {
'success': False,
'message': ['Campaign already exists.'],
'id': str(campaign.id)
}
# Create new campaign.
campaign = Campaign(name=name)
campaign.edit_description(description)
if bucket_list:
campaign.add_bucket_list(bucket_list, analyst)
if ticket:
campaign.add_ticket(ticket, analyst)
# Adjust aliases.
if isinstance(aliases, basestring):
alias_list = aliases.split(',')
final_aliases = [a.strip() for a in alias_list]
elif isinstance(aliases, list):
final_aliases = [a.strip() for a in aliases]
else:
final_aliases = []
campaign.add_alias(final_aliases)
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
campaign.save(username=analyst)
if related_obj and relationship_type and campaign:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
campaign.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
campaign.save(username=analyst)
campaign.reload()
try:
campaign.save(username=analyst)
campaign.reload()
return {
'success': True,
'message': 'Campaign created successfully!',
'id': str(campaign.id)
}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def process_changed_fields(initial_message, changed_fields, obj):
"""
Processes the changed fields to determine what actually changed.
:param message: An initial message to include.
:type message: str
:param changed_fields: A list of field names that were changed.
:type changed_fields: list of str
:param obj: The object.
:type obj: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:returns: str: Returns a message indicating what was changed.
"""
obj_type = obj._meta['crits_type']
message = initial_message
if message is None:
message = ''
source_filter = None
for changed_field in changed_fields:
# Fields may be fully qualified, e.g. source.1.instances.0.reference
# So, split on the '.' character and get the root of the changed field
base_changed_field = MappedMongoFields.get_mapped_mongo_field(
obj_type,
changed_field.split('.')[0])
new_value = getattr(obj, base_changed_field, '')
old_obj = class_from_id(obj_type, obj.id)
old_value = getattr(old_obj, base_changed_field, '')
change_handler = ChangeParser.get_changed_field_handler(
obj_type, base_changed_field)
if change_handler is not None:
change_message = change_handler(old_value, new_value,
base_changed_field)
if isinstance(change_message, dict):
if change_message.get('source_filter') is not None:
new_source_filter = change_message.get('source_filter')
source_filter = combine_source_filters(
source_filter, new_source_filter)
change_message = change_message.get('message')
if change_message is not None:
message += "\n" + change_message[:1].capitalize(
) + change_message[1:]
else:
change_field_handler = ChangeParser.generic_single_field_change_handler
if isinstance(old_value, BaseList):
list_value = None
if len(old_value) > 0:
list_value = old_value[0]
elif len(new_value) > 0:
list_value = new_value[0]
if isinstance(list_value, basestring):
change_field_handler = ChangeParser.generic_list_change_handler
elif isinstance(list_value, EmbeddedDocument):
change_field_handler = ChangeParser.generic_list_json_change_handler
change_message = change_field_handler(old_value, new_value,
base_changed_field)
if isinstance(change_message, dict):
if change_message.get('source_filter') is not None:
new_source_filter = change_message.get('source_filter')
combine_source_filters(source_filter, new_source_filter)
change_message = change_message.get('message')
if change_message is not None:
message += "\n" + change_message[:1].capitalize(
) + change_message[1:]
return {'message': message, 'source_filter': source_filter}
def handle_raw_data_file(data, source_name, user=None,
description=None, title=None, data_type=None,
tool_name=None, tool_version=None, tool_details=None,
link_id=None, method='', reference='', tlp='',
copy_rels=False, bucket_list=None, ticket=None,
related_id=None, related_type=None, relationship_type=None):
"""
Add RawData.
:param data: The data of the RawData.
:type data: str
:param source_name: The source which provided this RawData.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the RawData.
:type user: str
:param description: Description of the RawData.
:type description: str
:param title: Title of the RawData.
:type title: str
:param data_type: Datatype of the RawData.
:type data_type: str
:param tool_name: Name of the tool used to acquire/generate the RawData.
:type tool_name: str
:param tool_version: Version of the tool.
:type tool_version: str
:param tool_details: Details about the tool.
:type tool_details: str
:param link_id: LinkId to tie this to another RawData as a new version.
:type link_id: str
:param method: The method of acquiring this RawData.
:type method: str
:param reference: A reference to the source of this RawData.
:type reference: str
:param tlp: TLP for the source.
:type tlp: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this RawData
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this RawData
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
rdt = RawDataType.objects(name=data_type).first()
if not rdt:
status = {
'success': False,
'message': 'Invalid data type passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if isinstance(data, unicode):
data=data.encode('utf-8')
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate raw_data
is_rawdata_new = False
raw_data = RawData.objects(md5=md5).first()
if not raw_data:
raw_data = RawData()
raw_data.created = timestamp
raw_data.description = description
raw_data.md5 = md5
# raw_data.source = [source_name]
raw_data.data = data
raw_data.title = title
raw_data.data_type = data_type
raw_data.add_tool(name=tool_name,
version=tool_version,
details=tool_details)
is_rawdata_new = True
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
source = create_embedded_source(source_name,
method=method,
reference=reference,
tlp=tlp,
analyst=user.username)
raw_data.add_source(source)
else:
return {"success":False,
"message": "User does not have permission to add object using source %s." % source_name}
# this will handle adding a new source, or an instance automatically
elif isinstance(source_name, EmbeddedSource):
raw_data.add_source(source_name, method=method, reference=reference, tlp=tlp, analyst=user.usrname)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
raw_data.add_source(s, method=method, reference=reference, tlp=tlp, analyst=user.username)
#XXX: need to validate this is a UUID
if link_id:
raw_data.link_id = link_id
if copy_rels:
rd2 = RawData.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
raw_data.save(username=user)
raw_data.reload()
for rel in rd2.relationships:
# Get object to relate to.
rel_item = class_from_id(rel.rel_type, rel.object_id)
if rel_item:
raw_data.add_relationship(rel_item,
rel.relationship,
rel_date=rel.relationship_date,
analyst=user.username)
raw_data.version = len(RawData.objects(link_id=link_id)) + 1
if bucket_list:
raw_data.add_bucket_list(bucket_list, user)
if ticket:
raw_data.add_ticket(ticket, user);
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
raw_data.save(username=user.username)
if related_obj and relationship_type and raw_data:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
raw_data.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
raw_data.save(username=user.username)
raw_data.reload()
# save raw_data
raw_data.save(username=user.username)
# run raw_data triage
if is_rawdata_new:
raw_data.reload()
run_triage(raw_data, user)
status = {
'success': True,
'message': 'Uploaded raw_data',
'_id': raw_data.id,
'object': raw_data
}
return status
def handle_signature_file(data,
source_name,
user=None,
description=None,
title=None,
data_type=None,
data_type_min_version=None,
data_type_max_version=None,
data_type_dependency=None,
link_id=None,
method='',
reference='',
copy_rels=False,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add Signature.
:param data: The data of the Signature.
:type data: str
:param source_name: The source which provided this Signature.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Signature.
:type user: str
:param description: Description of the Signature.
:type description: str
:param title: Title of the Signature.
:type title: str
:param data_type: Datatype of the Signature.
:type data_type: str
:param data_type: Datatype of the Signature.
:type data_type_min_version: str
:param data_type_min_version: Datatype tool minimum version.
:type data_type_max_version: str
:param data_type_max_version: Datatype tool maximum version.
:type data_type_dependency: list
:param data_type_dependency: Datatype tool dependency to be run
:param link_id: LinkId to tie this to another Signature as a new version.
:type link_id: str
:param method: The method of acquiring this Signature.
:type method: str
:param reference: A reference to the source of this Signature.
:type reference: str
:param copy_rels: Copy relationships from the previous version to this one.
:type copy_rels: bool
:param bucket_list: Bucket(s) to add to this Signature
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Signature
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'_id' (str) if successful.
"""
if not data or not title or not data_type:
status = {
'success': False,
'message': 'No data object, title, or data type passed in'
}
return status
if not source_name:
return {"success": False, "message": "Missing source information."}
rdt = SignatureType.objects(name=data_type).first()
if not rdt:
status = {'success': False, 'message': 'Invalid data type passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate signature
signature = Signature()
signature.created = timestamp
signature.description = description
signature.md5 = md5
signature.data = data
signature.title = title
signature.data_type = data_type
signature.data_type_min_version = data_type_min_version
signature.data_type_max_version = data_type_max_version
if data_type_dependency:
if type(data_type_dependency) == unicode:
data_type_dependency = data_type_dependency.split(",")
for item in data_type_dependency:
if item:
item = item.strip()
signature.data_type_dependency.append(str(item))
else:
data_type_dependency = []
# generate new source information and add to sample
if isinstance(source_name, basestring) and len(source_name) > 0:
source = create_embedded_source(source_name,
date=timestamp,
method=method,
reference=reference,
analyst=user)
# this will handle adding a new source, or an instance automatically
signature.add_source(source)
elif isinstance(source_name, EmbeddedSource):
signature.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
signature.add_source(s, method=method, reference=reference)
signature.version = len(Signature.objects(link_id=link_id)) + 1
if link_id:
signature.link_id = link_id
if copy_rels:
rd2 = Signature.objects(link_id=link_id).first()
if rd2:
if len(rd2.relationships):
signature.save(username=user)
signature.reload()
for rel in rd2.relationships:
# Get object to relate to.
rel_item = class_from_id(rel.rel_type, rel.object_id)
if rel_item:
signature.add_relationship(
rel_item,
rel.relationship,
rel_date=rel.relationship_date,
analyst=user)
if bucket_list:
signature.add_bucket_list(bucket_list, user)
if ticket:
signature.add_ticket(ticket, user)
related_obj = None
if related_id and related_type:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
signature.save(username=user)
if related_obj and signature and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
signature.add_relationship(related_obj,
relationship_type,
analyst=user,
get_rels=False)
signature.save(username=user)
signature.reload()
# save signature
signature.save(username=user)
signature.reload()
status = {
'success': True,
'message': 'Uploaded signature',
'_id': signature.id,
'object': signature
}
return status
def add_screenshot(description, tags, source, method, reference, analyst,
screenshot, screenshot_ids, oid, otype):
"""
Add a screenshot or screenshots to a top-level object.
:param description: The description of the screenshot.
:type description: str
:param tags: Tags associated with this screenshot.
:type tags: str, list
:param source: The source who provided the screenshot.
:type source: str
:param method: The method of acquiring this screenshot.
:type method: str
:param reference: A reference to the source of this screenshot.
:type reference: str
:param analyst: The user adding the screenshot.
:type analyst: str
:param screenshot: The screenshot to add.
:type screenshot: file handle
:param screenshot_ids: A list of ObjectIds of existing screenshots to add.
:type screenshot_ids: str, list
:param oid: The ObjectId of the top-level object to add to.
:type oid: str
:param otype: The top-level object type.
:type otype: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'id' (str) if successful,
'html' (str) if successful,
"""
result = {'success': False}
if not source:
result['message'] = "Must provide a source"
return result
obj = class_from_id(otype, oid)
if not obj:
result['message'] = "Could not find the top-level object."
return result
final_screenshots = []
if screenshot_ids:
if not isinstance(screenshot_ids, list):
screenshot_list = screenshot_ids.split(',')
else:
screenshot_list = screenshot_ids
for screenshot_id in screenshot_list:
screenshot_id = screenshot_id.strip().lower()
s = Screenshot.objects(id=screenshot_id).first()
if s:
s.add_source(source=source,
method=method,
reference=reference,
analyst=analyst)
s.add_tags(tags)
s.save()
obj.screenshots.append(screenshot_id)
obj.save()
final_screenshots.append(s)
else:
md5 = hashlib.md5(screenshot.read()).hexdigest()
check = Screenshot.objects(md5=md5).first()
if check:
s = check
s.add_tags(tags)
else:
s = Screenshot()
s.analyst = analyst
s.description = description
s.md5 = md5
screenshot.seek(0)
s.add_screenshot(screenshot, tags)
s.add_source(source=source,
method=method,
reference=reference,
analyst=analyst)
if not s.screenshot and not s.thumb:
result[
'message'] = "Problem adding screenshot to GridFS. No screenshot uploaded."
return result
try:
s.save(username=analyst)
final_screenshots.append(s)
except Exception, e:
result['message'] = str(e)
return result
obj.screenshots.append(str(s.id))
obj.save(username=analyst)
def get_notification_details(request, newer_than):
"""
Generate the data to render the notification dialogs.
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param newer_than: A filter that specifies that only notifications
newer than this time should be returned.
:type newer_than: str in ISODate format.
:returns: arguments (dict)
"""
username = request.user.username
notifications_list = []
notifications = None
latest_notification_time = None
lock = NotificationLockManager.get_notification_lock(username)
timeout = 0
# Critical section, check if there are notifications to be consumed.
lock.acquire()
try:
notifications = get_user_notifications(username, newer_than=newer_than)
if len(notifications) > 0:
latest_notification_time = str(notifications[0].created)
else:
# no new notifications -- block until time expiration or lock release
lock.wait(60)
# lock was released, check if there is any new information yet
notifications = get_user_notifications(username,
newer_than=newer_than)
if len(notifications) > 0:
latest_notification_time = str(notifications[0].created)
finally:
lock.release()
if latest_notification_time is not None:
acknowledgement_type = request.user.get_preference(
'toast_notifications', 'acknowledgement_type', 'sticky')
if acknowledgement_type == 'timeout':
timeout = request.user.get_preference('toast_notifications',
'timeout', 30) * 1000
for notification in notifications:
obj = class_from_id(notification.obj_type, notification.obj_id)
if obj is not None:
link_url = obj.get_details_url()
header = generate_notification_header(obj)
else:
if notification.header is not None:
header = notification.header
else:
header = "%s %s" % (notification.obj_type, notification.obj_id)
if notification.link_url is not None:
link_url = notification.link_url
else:
link_url = None
notification_type = notification.notification_type
if notification_type is None or notification_type not in NotificationType.ALL:
notification_type = NotificationType.ALERT
notification_data = {
"header":
header,
"message":
notification.notification,
"date_modified":
str(notification.created.strftime("%Y/%m/%d %H:%M:%S")),
"link":
link_url,
"modified_by":
notification.analyst,
"id":
str(notification.id),
"type":
notification_type,
}
notifications_list.append(notification_data)
return {
'notifications': notifications_list,
'newest_notification': latest_notification_time,
'server_time':
str(datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S")),
'timeout': timeout,
}
def ip_add_update(ip_address,
ip_type,
source=None,
source_method='',
source_reference='',
campaign=None,
confidence='low',
analyst=None,
is_add_indicator=False,
indicator_reference='',
bucket_list=None,
ticket=None,
is_validate_only=False,
cache={},
related_id=None,
related_type=None,
relationship_type=None,
description=''):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding/updating this IP.
:type analyst: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:param description: A description for this IP
:type description: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
if not source:
return {"success": False, "message": "Missing source information."}
(ip_address, error) = validate_and_normalize_ip(ip_address, ip_type)
if error:
return {"success": False, "message": error}
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if not ip_object.description:
ip_object.description = description or ''
elif ip_object.description != description:
ip_object.description += "\n" + (description or '')
if isinstance(source, basestring):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)
]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
if bucket_list:
ip_object.add_bucket_list(bucket_list, analyst)
if ticket:
ip_object.add_ticket(ticket, analyst)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=analyst)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' %
(resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
handle_indicator_ind(ip_address,
source,
ip_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
method=source_method,
reference=indicator_reference,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if related_obj and ip_object and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
ip_object.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
ip_object.save(username=analyst)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(ip_object, analyst)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def add_object(type_, oid, object_type, name, source, method,
reference, analyst, value=None, file_=None,
add_indicator=False, get_objects=True,
indicator_campaign=None, indicator_campaign_confidence=None,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
name,
value,
source,
method,
reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
parent_id=oid,
parent_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method=method,
campaign=indicator_campaign,
campaign_confidence=indicator_campaign_confidence,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
# Inherit campaigns from top level item when creating
# an indicator from an object if no campaigns were specified
if indicator_campaign == None and ind != None:
for campaign in obj.campaign:
ec = EmbeddedCampaign(name=campaign.name,
confidence=campaign.confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
ind.add_campaign(ec)
ind.save(username=analyst)
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
return results
except ValidationError, e:
return {'success': False,
'message': e}
def upsert_target(data, analyst):
"""
Add/update target information.
:param data: The target information.
:type data: dict
:param analyst: The user adding the target.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if 'email_address' not in data:
return {'success': False, 'message': "No email address to look up"}
# check for exact match first
target = Target.objects(email_address=data['email_address']).first()
if not target: # if no exact match, look for case-insensitive match
target = Target.objects(
email_address__iexact=data['email_address']).first()
is_new = False
if not target:
is_new = True
target = Target()
target.email_address = data['email_address'].strip().lower()
bucket_list = False
ticket = False
related_id = False
if 'department' in data:
target.department = data['department']
if 'division' in data:
target.division = data['division']
if 'organization_id' in data:
target.organization_id = data['organization_id']
if 'firstname' in data:
target.firstname = data['firstname']
if 'lastname' in data:
target.lastname = data['lastname']
if 'note' in data:
target.note = data['note']
if 'title' in data:
target.title = data['title']
if 'campaign' in data and 'camp_conf' in data:
target.add_campaign(
EmbeddedCampaign(name=data['campaign'],
confidence=data['camp_conf'],
analyst=analyst))
if 'bucket_list' in data:
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
if 'ticket' in data:
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if 'related_id' in data:
related_id = data['related_id']
if 'related_type' in data:
related_type = data['related_type']
if 'relationship_type' in data:
relationship_type = data['relationship_type']
if bucket_list:
target.add_bucket_list(bucket_list, analyst)
if ticket:
target.add_ticket(ticket, analyst)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
try:
target.save(username=analyst)
if related_obj and target:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
target.add_relationship(related_obj,
relationship_type,
analyst=analyst,
get_rels=False)
target.save(username=analyst)
target.reload()
if is_new:
run_triage(target, analyst)
return {
'success': True,
'message': "Target saved successfully",
'id': str(target.id)
}
except ValidationError, e:
return {'success': False, 'message': "Target save failed: %s" % e}
def add_object(type_, oid, object_type, name, source, method,
reference, analyst, value=None, file_=None,
add_indicator=False, get_objects=True,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
if name == "URL" and "://" not in value.split('.')[0]:
return {"success" : False, "message" : "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"}
elif object_type == "Address":
if "ipv4" in name:
try:
validate_ipv4_address(value)
except DjangoValidationError:
return {"success" : False, "message" : "Invalid IPv4 address. "}
elif "ipv6" in name:
try:
validate_ipv6_address(value)
except DjangoValidationError:
return {"success" : False, "message" : "Invalid IPv6 address. "}
elif "cidr" in name:
try:
if '/' not in value:
raise ValidationError("")
cidr_parts = value.split('/')
if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128:
raise ValidationError("")
if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32:
raise ValidationError("")
validate_ipv46_address(cidr_parts[0])
except (ValidationError, ValueError) as cidr_error:
return {"success" : False, "message" : "Invalid CIDR address. "}
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
name,
value,
source,
method,
reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
related_id=oid,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False,
'message': str(e)}
def handle_cert_file(filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method='',
reference='',
relationship=None,
bucket_list=None,
ticket=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_id: ObjectId of a top-level object related to this Certificate.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param reference: A reference to the source of this Certificate.
:type reference: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
if ((related_type and not (related_id or related_md5))
or (not related_type and (related_id or related_md5))):
status = {
'success':
False,
'message':
'Must specify both related_type and related_id or related_md5.'
}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {'success': False, 'message': 'Related object not found.'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference=reference,
analyst=user)
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s, method=method, reference=reference)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if len(AnalysisResult.objects(object_id=str(cert.id))) < 1 and data:
run_triage(cert, user)
# update relationship if a related top-level object is supplied
if related_obj and cert:
if not relationship:
relationship = "Related_To"
cert.add_relationship(related_obj,
relationship,
analyst=user,
get_rels=False)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
'id': str(cert.id),
'object': cert
}
return status
def update_relationship_reasons(left_class=None, right_class=None,
left_type=None, left_id=None,
right_type=None, right_id=None,
rel_type=None, rel_date=None,
new_type=None,analyst=None, new_reason="N/A"):
"""
Update the relationship type between two top-level objects.
:param left_class: The first top-level object.
:type left_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param right_class: The second top-level object.
:type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param left_type: The type of first top-level object.
:type left_type: str
:param left_id: The ObjectId of the first top-level object.
:type left_id: str
:param right_type: The type of second top-level object.
:type right_type: str
:param right_id: The ObjectId of the second top-level object.
:type right_id: str
:param rel_type: The type of relationship.
:type rel_type: str
:param rel_date: The date this relationship applies.
:type rel_date: datetime.datetime
:param analyst: The user updating this relationship.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if rel_date is None or rel_date == 'None':
rel_date = None
elif isinstance(rel_date, basestring) and rel_date != '':
rel_date = parse(rel_date, fuzzy=True)
elif not isinstance(rel_date, datetime.datetime):
rel_date = None
if not left_class:
if left_type and left_id:
left_class = class_from_id(left_type, left_id)
else:
return {'success': False,
'message': "Need a valid left type and id"}
# update relationship
if right_class:
results = left_class.edit_relationship_reason(rel_item=right_class,
rel_type=rel_type,
rel_date=rel_date,
new_reason=new_reason,
analyst=analyst)
left_class.save(username=analyst)
right_class.save(username=analyst)
else:
if right_type and right_id:
results = left_class.edit_relationship_reason(type_=right_type,
rel_id=right_id,
rel_type=rel_type,
rel_date=rel_date,
new_reason=new_reason,
analyst=analyst)
left_class.save(username=analyst)
else:
return {'success': False,
'message': "Need a valid right type and id"}
return results
def add_new_handler_object(data, rowData, request, errors, is_validate_only=False,
is_sort_relationships=False, cache={}, obj=None):
"""
Add an object to the database.
:param data: The data for the object.
:type data: dict
:param rowData: Data from the row if using mass object upload.
:type rowData: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: List of existing errors to append to.
:type errors: list
:param is_validate_only: Only validate.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:returns: tuple of (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
username = request.user.username
if data:
object_type = data.get('object_type')
value = data.get('value')
source = data.get('source')
method = data.get('method')
reference = data.get('reference')
otype = data.get('otype')
oid = data.get('oid')
add_indicator = data.get('add_indicator')
elif rowData:
object_type = rowData.get(form_consts.Object.OBJECT_TYPE)
value = rowData.get(form_consts.Object.VALUE)
source = rowData.get(form_consts.Object.SOURCE)
method = rowData.get(form_consts.Object.METHOD)
reference = rowData.get(form_consts.Object.REFERENCE)
otype = rowData.get(form_consts.Object.PARENT_OBJECT_TYPE)
oid = rowData.get(form_consts.Object.PARENT_OBJECT_ID)
add_indicator = rowData.get(form_consts.Object.ADD_INDICATOR)
is_validate_locally = False
analyst = "%s" % username
# Default the user source to the user's organization if not specified
if not source:
source = cache.get('object_user_source')
if not source:
source = get_user_organization(analyst)
cache['object_user_source'] = source
ot_array = object_type.split(" - ")
object_type = ot_array[0]
name = ot_array[1] if len(ot_array) == 2 else ot_array[0]
if (otype == "" or otype == None) or (oid == "" or oid == None):
is_validate_locally = True
# TODO file_
object_result = add_object(otype, oid, object_type, name,
source, method, reference, analyst, value=value,
file_=None, add_indicator=add_indicator, get_objects=False,
obj=obj, is_validate_only=is_validate_only, is_sort_relationships=is_sort_relationships,
is_validate_locally=is_validate_locally, cache=cache)
if object_result['success'] == True:
if is_validate_only == False:
result = True
if obj == None:
obj = class_from_id(otype, oid)
if obj:
retVal['secondary'] = {'type': otype, 'id': oid}
if object_result.get('relationships'):
retVal['secondary']['relationships'] = object_result.get('relationships')
else:
errors += object_result['message']
return result, errors, retVal
def add_object(type_, id_, object_type, source, method, reference, tlp, user,
value=None, file_=None, add_indicator=False, get_objects=True,
tlo=None, is_sort_relationships=False, is_validate_only=False,
is_validate_locally=False, cache={}, **kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_objects: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_sort_relationships: Return all relationships and meta, sorted
:type is_sort_relationships: bool
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
# if object_type is a validated indicator type, then validate value
if value:
from crits.indicators.handlers import validate_indicator_value
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no TLO provided
return {"success": True}
if not tlo:
if type_ and id_:
tlo = class_from_id(type_, id_)
if not tlo:
return {'success': False, 'message': "Failed to find TLO"}
try:
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
ret = tlo.add_object(object_type, value,
source, method, reference, user)
if not ret['success']:
msg = '%s! [Type: "%s"][Value: "%s"]'
return {"success": False,
"message": msg % (ret['message'], object_type, value)}
else:
results = {'success': True}
if not is_validate_only: # save the object
tlo.update(add_to_set__obj=ret['object'])
results['message'] = "Object added successfully"
if file_:
# do we have a pcap?
if detect_pcap(data):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and not is_validate_only:
campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
from crits.indicators.handlers import handle_indicator_ind
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
forge_relationship(class_=tlo,
right_class=ind_res['object'],
rel_type=RelationshipTypes.RELATED_TO,
user=user)
else:
msg = "Object added, but failed to add Indicator.<br>Error: %s"
results['message'] = msg % ind_res.get('message')
if is_sort_relationships == True:
results['relationships'] = tlo.sort_relationships(user, meta=True)
if get_objects:
results['objects'] = tlo.sort_objects()
results['id'] = str(tlo.id)
return results
except ValidationError as e:
return {'success': False, 'message': str(e)}
def run(self, obj, config):
obj.filedata.seek(0)
data8 = obj.filedata.read(8)
obj.filedata.seek(0)
user = self.current_task.user
self.config = config
self.obj = obj
self._debug("pdf2txt started")
pdf2txt_path = self.config.get("pdf2txt_path", "/usr/bin/pdftotext")
antiword_path = self.config.get("antiword_path", "/usr/bin/antiword")
# The _write_to_file() context manager will delete this file at the
# end of the "with" block.
with self._write_to_file() as tmp_file:
(working_dir, filename) = os.path.split(tmp_file)
new_env = dict(os.environ) # Copy current environment
args = []
obj.filedata.seek(0)
if obj.is_pdf():
self._debug("PDF")
args = [pdf2txt_path, filename, "-"]
elif data8.startswith("\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
self._debug("Word")
#new_env['LANG'] = 'en_US'
#env=dict(os.environ, LANG="en_US")
args = [antiword_path, '-r', '-s', '-t', filename]
else:
self._error("Not a valid PDF or Word document")
return False
if not user.has_access_to(RawDataACL.WRITE):
self._info("User does not have permission to add Raw Data to CRITs")
self._add_result("Parsing Cancelled", "User does not have permission to add Raw Data to CRITs")
return
# pdftotext does not generate a lot of output, so we should not have to
# worry about this hanging because the buffer is full
proc = subprocess.Popen(args, env=new_env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, cwd=working_dir)
# Note that we are redirecting STDERR to STDOUT, so we can ignore
# the second element of the tuple returned by communicate().
output, serr = proc.communicate()
if serr:
self._warning(serr)
if proc.returncode:
msg = ("pdftotext could not process the file.")
self._warning(msg)
return
raw_hash = md5(output).hexdigest()
res = handle_raw_data_file(output, self.obj.source, self.current_task.user,
title="pdftotext", data_type='Text',
tool_name='pdftotext', tool_version='0.1', tool_details='http://poppler.freedesktop.org',
method=self.name,
copy_rels=True)
raw_obj = class_from_id("RawData", res["_id"])
self._warning("obj.id: %s, raw_id:%s, suc: %s" % (str(obj.id), str(raw_obj.id), repr(res['success']) ) )
# update relationship if a related top-level object is supplied
rel_type = RelationshipTypes.RELATED_TO
if obj.id != raw_obj.id: #don't form relationship to itself
resy = obj.add_relationship(rel_item=raw_obj,
rel_type=rel_type,
rel_date=datetime.now(),
analyst=self.current_task.user)
obj.save(username=self.current_task.user.username)
raw_obj.save(username=self.current_task.user.username)
self._warning("resy: %s" % (str(resy)) )
self._add_result("rawdata_added", raw_hash, {'md5': raw_hash})
return
def campaign_add(campaign_name,
confidence,
description,
related,
analyst,
ctype=None,
oid=None,
obj=None,
update=True):
"""
Attribute a Campaign to a top-level object.
:param campaign_name: The Campaign to attribute.
:type campaign_name: str
:param confidence: The confidence level of this attribution (low, medium, high)
:type confidence: str
:param description: Description of this attribution.
:type description: str
:param related: Should this attribution propagate to related top-level objects.
:type related: boolean
:param analyst: The user attributing this Campaign.
:type analyst: str
:param ctype: The top-level object type.
:type ctype: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param obj: The top-level object instantiated class.
:type obj: Instantiated class object
:param update: If True, allow merge with pre-existing campaigns
: If False, do not change any pre-existing campaigns
:type update: boolean
:returns: dict with keys:
'success' (boolean),
'html' (str) if successful,
'message' (str).
"""
if not obj:
if ctype and oid:
# Verify the document exists.
obj = class_from_id(ctype, oid)
if not obj:
return {'success': False, 'message': 'Cannot find %s.' % ctype}
else:
return {
'success':
False,
'message':
'Object type and ID, or object instance, must be provided.'
}
# Create the embedded campaign.
campaign = EmbeddedCampaign(name=campaign_name,
confidence=confidence,
description=description,
analyst=analyst)
result = obj.add_campaign(campaign, update=update)
if result['success']:
if related:
campaign_addto_related(obj, campaign, analyst)
try:
obj.save(username=analyst)
html = obj.format_campaign(campaign, analyst)
return {
'success': True,
'html': html,
'message': result['message']
}
except ValidationError, e:
return {'success': False, 'message': "Invalid value: %s" % e}
def create_indicator_from_object(rel_type, rel_id, ind_type, value,
source_name, method, reference, analyst, request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param source_name: The source name for the indicator.
:type source_name: str
:param method: The source method for the indicator.
:type method: str
:param reference: The source reference for the indicator.
:type reference: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False,
'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty value field"}
elif ind_type == None or ind_type.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty type field"}
elif source_name == None or source_name.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty source field"}
else:
value = value.lower().strip()
ind_type = ind_type.strip()
source_name = source_name.strip()
create_indicator_result = {}
ind_tlist = ind_type.split(" - ")
if ind_tlist[0] == ind_tlist[1]:
ind_type = ind_tlist[0]
from crits.indicators.handlers import handle_indicator_ind
campaign = me.campaign if hasattr(me, 'campaign') else None
create_indicator_result = handle_indicator_ind(value,
source_name,
reference,
ind_type,
analyst,
method,
add_domain=True,
campaign=campaign)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return create_indicator_result
indicator = Indicator.objects(ind_type=ind_type,
value=value).first()
if not indicator:
result = {'success': False,
'message': "Could not create indicator"}
else:
results = me.add_relationship(rel_item=indicator,
rel_type="Related_To",
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
indicator.save(username=analyst)
relationship= {'type': rel_type, 'value': rel_id}
message = render_to_string('relationships_listing_widget.html',
{'relationship': relationship,
'nohide': True,
'relationships': results['message']},
RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False,
'message': message}
return result
def create_indicator_from_object(rel_type, rel_id, ind_type, value,
source_name, method, reference, tlp, analyst, request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param source_name: The source name for the indicator.
:type source_name: str
:param method: The source method for the indicator.
:type method: str
:param reference: The source reference for the indicator.
:type reference: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False,
'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty value field"}
elif ind_type == None or ind_type.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty type field"}
elif source_name == None or source_name.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty source field"}
else:
value = value.lower().strip()
ind_type = ind_type.strip()
source_name = source_name.strip()
create_indicator_result = {}
from crits.indicators.handlers import handle_indicator_ind
campaign = me.campaign if hasattr(me, 'campaign') else None
create_indicator_result = handle_indicator_ind(value,
source_name,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return create_indicator_result
indicator = Indicator.objects(ind_type=ind_type,
value=value).first()
if not indicator:
result = {'success': False,
'message': "Could not create indicator"}
else:
results = me.add_relationship(indicator,
RelationshipTypes.RELATED_TO,
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
relationship= {'type': rel_type, 'value': rel_id}
message = render_to_string('relationships_listing_widget.html',
{'relationship': relationship,
'nohide': True,
'relationships': results['message']},
request=request)
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False,
'message': message}
return result
def run_service(name, type_, id_, user, obj=None,
execute='local', custom_config={}, is_triage_run=False, **kwargs):
"""
Run a service.
:param name: The name of the service to run.
:type name: str
:param type_: The type of the object.
:type type_: str
:param id_: The identifier of the object.
:type id_: str
:param user: The user running the service.
:type user: str
:param obj: The CRITs object, if given this overrides crits_type and identifier.
:type obj: CRITs object.
:param analyst: The user updating the results.
:type analyst: str
:param execute: The execution type.
:type execute: str
:param custom_config: Use a custom configuration for this run.
:type custom_config: dict
"""
result = {'success': False}
if type_ not in settings.CRITS_TYPES:
result['html'] = "Unknown CRITs type."
return result
if name not in enabled_services():
result['html'] = "Service %s is unknown or not enabled." % name
return result
service_class = crits.services.manager.get_service_class(name)
if not service_class:
result['html'] = "Unable to get service class."
return result
if not obj:
obj = class_from_id(type_, id_)
if not obj:
result['html'] = 'Could not find object.'
return result
service = CRITsService.objects(name=name).first()
if not service:
result['html'] = "Unable to find service in database."
return result
# See if the object is a supported type for the service.
if not service_class.supported_for_type(type_):
result['html'] = "Service not supported for type '%s'" % type_
return result
# When running in threaded mode, each thread needs to have its own copy of
# the object. If we do not do this then one thread may read() from the
# object (to get the binary) and then the second would would read() without
# knowing and get undefined behavior as the file pointer would be who knows
# where. By giving each thread a local copy they can operate independently.
#
# When not running in thread mode this has no effect except wasted memory.
local_obj = local()
local_obj.obj = copy.deepcopy(obj)
# Give the service a chance to check for required fields.
try:
service_class.valid_for(local_obj.obj)
if hasattr(local_obj.obj, 'filedata'):
if local_obj.obj.filedata.grid_id:
# Reset back to the start so the service gets the full file.
local_obj.obj.filedata.seek(0)
except ServiceConfigError as e:
result['html'] = str(e)
return result
# Get the config from the database and validate the submitted options
# exist.
db_config = service.config.to_dict()
try:
service_class.validate_runtime(custom_config, db_config)
except ServiceConfigError as e:
result['html'] = str(e)
return result
final_config = db_config
# Merge the submitted config with the one from the database.
# This is because not all config options may be submitted.
final_config.update(custom_config)
form = service_class.bind_runtime_form(user, final_config)
if form:
if not form.is_valid():
# TODO: return corrected form via AJAX
result['html'] = str(form.errors)
return result
# If the form is valid, create the config using the cleaned data.
final_config = db_config
final_config.update(form.cleaned_data)
logger.info("Running %s on %s, execute=%s" % (name, local_obj.obj.id, execute))
service_instance = service_class(notify=update_analysis_results,
complete=finish_task)
# Determine if this service is being run via triage
if is_triage_run:
service_instance.is_triage_run = True
# Give the service a chance to modify the config that gets saved to the DB.
saved_config = dict(final_config)
service_class.save_runtime_config(saved_config)
task = AnalysisTask(local_obj.obj, service_instance, user)
task.config = AnalysisConfig(**saved_config)
task.start()
add_task(task)
service_instance.set_task(task)
if execute == 'process':
p = Process(target=service_instance.execute, args=(final_config,))
p.start()
elif execute == 'thread':
t = Thread(target=service_instance.execute, args=(final_config,))
t.start()
elif execute == 'process_pool':
if __service_process_pool__ is not None and service.compatability_mode != True:
__service_process_pool__.apply_async(func=service_work_handler,
args=(service_instance, final_config,))
else:
logger.warning("Could not run %s on %s, execute=%s, running in process mode" % (name, local_obj.obj.id, execute))
p = Process(target=service_instance.execute, args=(final_config,))
p.start()
elif execute == 'thread_pool':
if __service_thread_pool__ is not None and service.compatability_mode != True:
__service_thread_pool__.apply_async(func=service_work_handler,
args=(service_instance, final_config,))
else:
logger.warning("Could not run %s on %s, execute=%s, running in thread mode" % (name, local_obj.obj.id, execute))
t = Thread(target=service_instance.execute, args=(final_config,))
t.start()
elif execute == 'local':
service_instance.execute(final_config)
# Return after starting thread so web request can complete.
result['success'] = True
return result
def forge_relationship(left_class=None,
right_class=None,
left_type=None,
left_id=None,
right_type=None,
right_id=None,
rel_type=None,
rel_date=None,
analyst=None,
rel_reason="N/A",
rel_confidence='unknown',
get_rels=False):
"""
Forge a relationship between two top-level objects.
:param left_class: The first top-level object to relate to.
:type left_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param right_class: The second top-level object to relate to.
:type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param left_type: The type of first top-level object to relate to.
:type left_type: str
:param left_id: The ObjectId of the first top-level object.
:type left_id: str
:param right_type: The type of second top-level object to relate to.
:type right_type: str
:param right_id: The ObjectId of the second top-level object.
:type right_id: str
:param rel_type: The type of relationship.
:type rel_type: str
:param rel_date: The date this relationship applies.
:type rel_date: datetime.datetime
:param analyst: The user forging this relationship.
:type analyst: str
:param rel_reason: The reason for the relationship.
:type rel_reason: str
:param rel_confidence: The confidence of the relationship.
:type rel_confidence: str
:param get_rels: Return the relationships after forging.
:type get_rels: boolean
:returns: dict with keys "success" (boolean) and "message" (str if
failed, dict if successful)
"""
if rel_date is None or rel_date == 'None':
rel_date = None
elif isinstance(rel_date, basestring) and rel_date != '':
rel_date = parse(rel_date, fuzzy=True)
elif not isinstance(rel_date, datetime.datetime):
rel_date = None
if not left_class:
if left_type and left_id:
left_class = class_from_id(left_type, left_id)
if not left_class:
return {'success': False, 'message': "Unable to get object."}
else:
return {
'success': False,
'message': "Need a valid left type and id"
}
try:
# forge relationship
if right_class:
results = left_class.add_relationship(
rel_item=right_class,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst,
rel_confidence=rel_confidence,
rel_reason=rel_reason)
right_class.save(username=analyst)
else:
if right_type and right_id:
results = left_class.add_relationship(
type_=right_type,
rel_id=right_id,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst,
rel_confidence=rel_confidence,
rel_reason=rel_reason)
else:
return {
'success': False,
'message': "Need a valid right type and id"
}
except Exception, e:
return {'success': False, 'message': e}
def ip_add_update(ip_address, ip_type, source=None, source_method='',
source_reference='', source_tlp=None, campaign=None,
confidence='low', user=None, is_add_indicator=False,
indicator_reference='', bucket_list=None, ticket=None,
is_validate_only=False, cache={}, related_id=None,
related_type=None, relationship_type=None, description=''):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding/updating this IP.
:type user: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:param description: A description for this IP
:type description: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
if not source:
return {"success" : False, "message" : "Missing source information."}
source_name = source
(ip_address, error) = validate_and_normalize_ip(ip_address, ip_type)
if error:
return {"success": False, "message": error}
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if not ip_object.description:
ip_object.description = description or ''
elif ip_object.description != description:
ip_object.description += "\n" + (description or '')
if isinstance(source_name, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username)]
else:
return {"success":False,
"message": "User does not have permission to add object \
using source %s." % source}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user.username)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if bucket_list:
ip_object.add_bucket_list(bucket_list, user.username)
if ticket:
ip_object.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=user.username)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
result = handle_indicator_ind(ip_address,
source_name,
ip_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=source_method,
source_reference = indicator_reference,
source_tlp = source_tlp,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if related_obj and ip_object and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
ip_object.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
ip_object.save(username=user.username)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(ip_object, user)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def create_notification(obj,
username,
message,
source_filter=None,
notification_type=NotificationType.ALERT):
"""
Generate a notification -- based on mongo obj.
:param obj: The object.
:type obj: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param username: The user creating the notification.
:type username: str
:param message: The notification message.
:type message: str
:param source_filter: Filter on who can see this notification.
:type source_filter: list(str)
:param notification_type: The notification type (e.g. alert, error).
:type notification_type: str
"""
n = Notification()
n.analyst = username
obj_type = obj._meta['crits_type']
users = set()
if notification_type not in NotificationType.ALL:
notification_type = NotificationType.ALERT
n.notification_type = notification_type
if obj_type == 'Comment':
n.obj_id = obj.obj_id
n.obj_type = obj.obj_type
n.notification = "%s added a comment: %s" % (username, obj.comment)
users.update(obj.users) # notify mentioned users
# for comments, use the sources from the object that it is linked to
# instead of the comments's sources
obj = class_from_id(n.obj_type, n.obj_id)
else:
n.notification = message
n.obj_id = obj.id
n.obj_type = obj_type
if hasattr(obj, 'source'):
sources = [s.name for s in obj.source]
subscribed_users = get_subscribed_users(n.obj_type, n.obj_id, sources)
# Filter on users that have access to the source of the object
for subscribed_user in subscribed_users:
allowed_sources = user_sources(subscribed_user)
for allowed_source in allowed_sources:
if allowed_source in sources:
if source_filter is None or allowed_source in source_filter:
users.add(subscribed_user)
break
else:
users.update(get_subscribed_users(n.obj_type, n.obj_id, []))
users.discard(username) # don't notify the user creating this notification
n.users = list(users)
if not len(n.users):
return
try:
n.save()
except ValidationError:
pass
# Signal potentially waiting threads that notification information is available
for user in n.users:
notification_lock = NotificationLockManager.get_notification_lock(user)
notification_lock.acquire()
try:
notification_lock.notifyAll()
finally:
notification_lock.release()
