def get_diffie_config(analyst, type_, id_, data=None):
"""
Return DiffieConfigForm for diffie service.
Must make sure user has source access to requested type and id.
:param analyst: The username.
:type analyst: str
:param type_: CRITs type.
:type type_: str
:param id_: CRITs ID.
:type id_: str
:param data: Dictionary with submitted data.
:type data: dict
:returns: DiffieConfigForm
"""
results = {'success': False}
sources = user_sources(analyst)
klass = class_from_type(type_)
obj = klass.objects(source__name__in=sources, id=id_).first()
if not obj:
results[
'message'] = "Either no data exists for this object or you do not have permission to view it."
return results
results['success'] = True
results['form'] = forms.DiffieConfigForm(type_=type_, id_=id_, data=data)
return results
def get_diffie_config(analyst, type_, id_, data=None):
"""
Return DiffieConfigForm for diffie service.
Must make sure user has source access to requested type and id.
:param analyst: The username.
:type analyst: str
:param type_: CRITs type.
:type type_: str
:param id_: CRITs ID.
:type id_: str
:param data: Dictionary with submitted data.
:type data: dict
:returns: DiffieConfigForm
"""
results = {'success': False}
sources = user_sources(analyst)
klass = class_from_type(type_)
obj = klass.objects(source__name__in=sources, id=id_).first()
if not obj:
results['message'] = "Either no data exists for this object or you do not have permission to view it."
return results
results['success'] = True
results['form'] = forms.DiffieConfigForm(type_=type_, id_=id_, data=data)
return results
def get_crits_type(self, c_obj):
"""
Get the class that the given cybox object should be interpreted as during import.
:param c_obj: A CybOX object.
:type c_obj: An instance of one of the various CybOX object classes.
:returns: The CRITs class to use to import the given CybOX object.
"""
if isinstance(c_obj, Address):
imp_type = "IP"
elif isinstance(c_obj, DomainName):
imp_type = "Domain"
elif isinstance(c_obj, Artifact):
imp_type = "RawData"
elif isinstance(c_obj, File) and c_obj.custom_properties and c_obj.custom_properties[0].name == "crits_type" and c_obj.custom_properties[0]._value == "Certificate":
imp_type = "Certificate"
elif isinstance(c_obj, File) and self.has_network_artifact(c_obj):
imp_type = "PCAP"
elif isinstance(c_obj, File):
imp_type = "Sample"
elif isinstance(c_obj, EmailMessage):
imp_type = "Email"
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
return class_from_type(imp_type)
def handle(self, *args, **options):
"""
Script Execution.
"""
buckets = {}
types = [
'Actor', 'Campaign', 'Certificate', 'Domain', 'Email', 'Event',
'Indicator', 'IP', 'PCAP', 'RawData', 'Sample', 'Target'
]
for otype in types:
klass = class_from_type(otype)
if not klass:
continue
objs = klass.objects().only('bucket_list')
for obj in objs:
for bucket in obj.bucket_list:
if not bucket:
continue # Avoid empty strings
if bucket not in buckets:
buckets[bucket] = Bucket()
buckets[bucket].name = bucket
setattr(buckets[bucket], otype, 1)
else:
buckets[bucket][otype] += 1
# Drop all existing buckets
Bucket.objects().delete()
for bucket in buckets.values():
bucket.save()
def filter_and_format_choices(choice_opts, item, _type):
"""
Given a list of CRITs options, filter out options matching the given item
and format those options for display as a choice in a multi-select box.
:param choice_opts A dict of CRITs objects in (id, json_repr) format
:param item The item being viewed for TAXII service
:param _type The type of objects represented in the choice_opts dict
"""
from .handlers import has_cybox_repr
ret_opts = [] # return storage array
item_type = item._meta['crits_type'] # get the type of the subject
choice_fmt = formats.get_format(_type) # get formatting option for current CRITs type
ind_crits_type = class_from_type("Indicator")._meta['crits_type']
for choice in choice_opts:
obj = choice_opts.get(choice)[1]
if _type == ind_crits_type and not has_cybox_repr(obj):
# this indicator can't currently be converted to CybOX, so don't offer as option in UI
continue
if item.id != obj.id or item_type != _type:
# only process if the item isn't the current context crits item
ret_opts.append((choice, choice_fmt.format(obj).encode('utf-8')))
return ret_opts
def _analysis_exists(self, context, service_name, version=None):
"""
Check for existing analysis results
If the item of `crits_type` identified by `identifier` contains an
analysis result produced by service_name, return True. If version is
specified, only return True if there is an existing analysis result
using this version or later.
"""
obj_class = class_from_type(context.crits_type)
query = self.get_db_query(context)
obj = obj_class.objects(__raw__=query).first()
for analysis in obj.analysis:
if analysis.service_name == service_name:
if version is None:
return True
try:
result_version = StrictVersion(analysis.version)
except:
result_version = 0
if result_version >= version:
return True
return False
def campaign_addto_related(crits_object, campaign, analyst):
"""
Add this Campaign to all related top-level objects.
:param crits_object: The top-level object to get relationships for.
:type crits_object: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param campaign: The campaign to add to all related top-level objects.
:type campaign: :class:`crits.core.crits_mongoengine.EmbeddedCampaign`
:param analyst: The user adding this Campaign to the related top-level objects.
:type analyst: str
"""
for r in crits_object.relationships:
klass = class_from_type(r.rel_type)
if not klass:
continue
robj = klass.objects(id=str(r.object_id)).first()
if not robj:
continue
robj.add_campaign(campaign)
try:
robj.save(username=analyst)
except ValidationError:
pass
def update_analysis_results(task):
"""
Update analysis results for this task.
"""
# If the task does not currently exist for the given sample in the
# database, add it.
obj_class = class_from_type(task.obj._meta['crits_type'])
obj = obj_class.objects(id=task.obj.id).first()
obj_id = obj.id
found = False
for a in obj.analysis:
if str(a.analysis_id) == task.task_id:
found = True
break
if not found:
logger.warning("Tried to update a task that didn't exist.")
insert_analysis_results(task)
else:
# Otherwise, update it.
ear = EmbeddedAnalysisResult()
tdict = task.to_dict()
tdict['analysis_id'] = tdict['id']
del tdict['id']
ear.merge(arg_dict=tdict)
obj_class.objects(id=obj_id,
analysis__id=task.task_id).update_one(set__analysis__S=ear)
def inner_collect(obj_type, obj_id, sources, depth):
if obj_id in objects:
return
klass = class_from_type(obj_type)
if not klass:
return
if hasattr(klass, 'source'):
obj = klass.objects(id=obj_id, source__name__in=sources).first()
else:
obj = klass.objects(id=obj_id).first()
if not obj:
return
objects[obj_id] = obj
if depth == 0:
return
depth -= 1
for r in obj.relationships:
#if r.rel_type=='Sample':
inner_collect(r.rel_type, str(r.object_id), sources, depth)
def _analysis_exists(self, context, service_name, version=None):
"""
Check for existing analysis results
If the item of `crits_type` identified by `identifier` contains an
analysis result produced by service_name, return True. If version is
specified, only return True if there is an existing analysis result
using this version or later.
"""
obj_class = class_from_type(context.crits_type)
query = self.get_db_query(context)
obj = obj_class.objects(__raw__=query).first()
for analysis in obj.analysis:
if analysis.service_name == service_name:
if version is None:
return True
try:
result_version = StrictVersion(analysis.version)
except:
result_version = 0
if result_version >= version:
return True
return False
def handle(self, *args, **options):
"""
Script Execution.
"""
sectors = {}
types = ['Actor', 'Campaign', 'Certificate', 'Domain', 'Email', 'Event',
'Indicator', 'IP', 'PCAP', 'RawData', 'Sample', 'Signature', 'Target']
for otype in types:
klass = class_from_type(otype)
if not klass:
continue
objs = klass.objects().only('sectors')
for obj in objs:
for sector in obj.sectors:
if not sector:
continue # Avoid empty strings
if sector not in sectors:
sectors[sector] = Sector()
sectors[sector].name = sector
setattr(sectors[sector], otype, 1)
else:
sectors[sector][otype] += 1
# Drop all existing sectors
Sector.objects().delete()
for sector in sectors.values():
sector.save()
def add_results(object_type, object_id, analysis_id, result, type_, subtype,
analyst):
"""
Add multiple results to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param result: The list of result to append.
:type result: list of str
:param type_: The list of result types.
:type type_: list of str
:param subtype: The list of result subtypes.
:type subtype: list of str
:param analyst: The user updating the results.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
res = {'success': False}
if not object_type or not object_id or not analysis_id:
res['message'] = "Must supply object id/type and analysis id."
return res
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
res['message'] = "Could not find object to add results to."
return res
if not(result and type_ and subtype):
res['message'] = "Need a result, type, and subtype to add a result."
return res
if not(len(result) == len(type_) == len(subtype)):
res['message'] = "result, type, and subtype need to be the same length."
return res
# Update analysis results
final_list = []
for key, r in enumerate(result):
final = {}
final['subtype'] = subtype[key]
final['result'] = r
tmp = ast.literal_eval(type_[key])
for k in tmp:
final[k] = tmp[k]
final_list.append(final)
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(push_all__results=final_list)
res['success'] = True
return res
def handle(self, *args, **options):
"""
Script Execution.
"""
sectors = {}
types = [
'Actor', 'Campaign', 'Certificate', 'Domain', 'Email', 'Event',
'Indicator', 'IP', 'PCAP', 'RawData', 'Sample', 'Signature',
'Target'
]
for otype in types:
klass = class_from_type(otype)
if not klass:
continue
objs = klass.objects().only('sectors')
for obj in objs:
for sector in obj.sectors:
if not sector:
continue # Avoid empty strings
if sector not in sectors:
sectors[sector] = Sector()
sectors[sector].name = sector
setattr(sectors[sector], otype, 1)
else:
sectors[sector][otype] += 1
# Drop all existing sectors
Sector.objects().delete_one()
for sector in sectors.values():
sector.save()
def handle(self, *args, **options):
"""
Script Execution.
"""
buckets = {}
types = ['Actor', 'Campaign', 'Certificate', 'Domain', 'Email', 'Event',
'Indicator', 'IP', 'PCAP', 'RawData', 'Sample', 'Target']
for otype in types:
klass = class_from_type(otype)
if not klass:
continue
objs = klass.objects().only('bucket_list')
for obj in objs:
for bucket in obj.bucket_list:
if not bucket:
continue # Avoid empty strings
if bucket not in buckets:
buckets[bucket] = Bucket()
buckets[bucket].name = bucket
setattr(buckets[bucket], otype, 1)
else:
buckets[bucket][otype] += 1
# Drop all existing buckets
Bucket.objects().delete()
for bucket in buckets.values():
bucket.save()
def campaign_addto_related(crits_object, campaign, analyst):
"""
Add this Campaign to all related top-level objects.
:param crits_object: The top-level object to get relationships for.
:type crits_object: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param campaign: The campaign to add to all related top-level objects.
:type campaign: :class:`crits.core.crits_mongoengine.EmbeddedCampaign`
:param analyst: The user adding this Campaign to the related top-level objects.
:type analyst: str
"""
for r in crits_object.relationships:
klass = class_from_type(r.rel_type)
if not klass:
continue
robj = klass.objects(id=str(r.object_id)).first()
if not robj:
continue
robj.add_campaign(campaign)
try:
robj.save(username=analyst)
except ValidationError:
pass
def add_results(object_type, object_id, analysis_id, result, type_, subtype,
analyst):
"""
Add multiple results to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param result: The list of result to append.
:type result: list of str
:param type_: The list of result types.
:type type_: list of str
:param subtype: The list of result subtypes.
:type subtype: list of str
:param analyst: The user updating the results.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
res = {'success': False}
if not object_type or not object_id or not analysis_id:
res['message'] = "Must supply object id/type and analysis id."
return res
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
res['message'] = "Could not find object to add results to."
return res
if not(result and type_ and subtype):
res['message'] = "Need a result, type, and subtype to add a result."
return res
if not(len(result) == len(type_) == len(subtype)):
res['message'] = "result, type, and subtype need to be the same length."
return res
# Update analysis results
final_list = []
for key, r in enumerate(result):
final = {}
final['subtype'] = subtype[key]
final['result'] = r
tmp = ast.literal_eval(type_[key])
for k in tmp:
final[k] = tmp[k]
final_list.append(final)
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(push_all__results=final_list)
res['success'] = True
return res
def get_crits_type(self, c_obj):
"""
Get the class that the given cybox object should be interpreted as during import.
:param c_obj: A CybOX object.
:type c_obj: An instance of one of the various CybOX object classes.
:returns: The CRITs class to use to import the given CybOX object.
"""
if isinstance(c_obj, Address):
imp_type = "IP"
elif isinstance(c_obj, DomainName):
imp_type = "Domain"
elif isinstance(c_obj, Artifact):
imp_type = "RawData"
elif isinstance(
c_obj,
File) and c_obj.custom_properties and c_obj.custom_properties[
0].name == "crits_type" and c_obj.custom_properties[
0]._value == "Certificate":
imp_type = "Certificate"
elif isinstance(c_obj, File) and self.has_network_artifact(c_obj):
imp_type = "PCAP"
elif isinstance(c_obj, File):
imp_type = "Sample"
elif isinstance(c_obj, EmailMessage):
imp_type = "Email"
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
return class_from_type(imp_type)
def filter_and_format_choices(choice_opts, item, _type):
"""
Given a list of CRITs options, filter out options matching the given item
and format those options for display as a choice in a multi-select box.
:param choice_opts A dict of CRITs objects in (id, json_repr) format
:param item The item being viewed for TAXII service
:param _type The type of objects represented in the choice_opts dict
"""
from .handlers import has_cybox_repr
ret_opts = [] # return storage array
item_type = item._meta['crits_type'] # get the type of the subject
choice_fmt = formats.get_format(
_type) # get the formatting option for the current CRITs type
for choice in choice_opts:
obj = choice_opts.get(choice)[1]
if _type == class_from_type(
"Indicator")._meta['crits_type'] and not has_cybox_repr(obj):
# this indicator can't currently be converted to CybOX, so don't offer as option in UI
continue
if item.id != obj.id or item_type != _type:
# only process if the item isn't the current context crits item
ret_opts.append((choice, choice_fmt.format(obj).encode('utf-8')))
return ret_opts
def add_result(object_type, object_id, analysis_id, result, type_, subtype,
analyst):
"""
Add a result to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param result: The result to append.
:type result: str
:param type_: The result type.
:type type_: str
:param subtype: The result subtype.
:type subtype: str
:param analyst: The user updating the results.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
res = {'success': False}
if not object_type or not object_id or not analysis_id:
res['message'] = "Must supply object id/type and analysis id."
return res
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
res['message'] = "Could not find object to add results to."
return res
found = False
c = 0
for a in obj.analysis:
if str(a.analysis_id) == analysis_id:
found = True
break
c += 1
if not found:
res['message'] = "Could not find an analysis task to update."
return res
if result and type_ and subtype:
final = {}
final['subtype'] = subtype
final['result'] = result
tmp = ast.literal_eval(type_)
for k in tmp:
final[k] = tmp[k]
klass.objects(id=object_id,
analysis__id=analysis_id).update_one(push__analysis__S__results=final)
else:
res['message'] = "Need a result, type, and subtype to add a result."
return res
res['success'] = True
return res
def get_supported_types():
"""
Get a list of supported types for TAXII service.
"""
supported_types = []
for ctype in settings.CRITS_TYPES:
cls = class_from_type(ctype)
if hasattr(cls, "to_stix_indicator") or hasattr(cls, "to_cybox_observable"):
supported_types.append(ctype)
return supported_types
def get_user_allowed_comments(comments, sources):
"""
Limit the comments to those a user should have access to see.
:param comments: The list of comments.
:type comments: list
:param sources: The sources the user has access to.
:type sources: list
:returns: list of :class:`crits.comments.comment.Comment`
"""
docs = {
'Actor': {},
'Campaign': {},
'Certificate': {},
'Domain': {},
'Email': {},
'Event': {},
'Indicator': {},
'IP': {},
'PCAP': {},
'RawData': {},
'Sample': {},
'Target': {}
}
for c in comments:
c.comment_to_html()
try:
docs[c.obj_type][c.obj_id].append(c)
except KeyError:
docs[c.obj_type][c.obj_id] = [c]
final_comments = []
for key, val in docs.items():
cls = class_from_type(key)
obj_ids = [v for v in val] #get keys
query = {
'_id': {
'$in': obj_ids
},
'$or': [{
'source.name': {
'$in': sources
}
}, {
'source': {
'$exists': 0
}
}]
}
result = cls.objects(__raw__=query).only('id')
for r in result:
final_comments += val[r.id]
final_comments.sort(key=lambda x: x.created, reverse=True)
return final_comments
def get_parent(self):
"""
Get the parent CRITs object.
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`.
"""
col_obj = class_from_type(self.obj_type)
doc = col_obj.objects(id=self.obj_id).first()
return doc
def get_supported_types():
"""
Get a list of supported types for TAXII service.
"""
supported_types = []
for ctype in settings.CRITS_TYPES:
cls = class_from_type(ctype)
if hasattr(cls, "to_stix_indicator") or hasattr(
cls, "to_cybox_observable"):
supported_types.append(ctype)
return supported_types
def inner_collect(obj_type, obj_id, sources, depth):
# Don't keep going if we've already processed this object
if obj_id in objects:
return
klass = class_from_type(obj_type)
if not klass:
return
if hasattr(klass, 'source'):
obj = klass.objects(id=obj_id, source__name__in=sources).first()
else:
obj = klass.objects(id=obj_id).first()
if not obj:
return
objects[obj_id] = obj
if depth == 0:
return
depth -= 1
for r in obj.relationships:
inner_collect(r.rel_type, str(r.object_id), sources, depth)
# If we traverse into a Campaign object, walk everything tagged
# with that campaign along with related objects.
if obj_type == 'Campaign':
for c in field_dict.keys():
klass = class_from_type(c)
# Not every object in field_dict can be tagged with a campaign.
# For example, comments.
if not hasattr(klass, 'campaign'):
continue
tagged_objs = klass.objects(campaign__name=obj.name)
for tobj in tagged_objs:
inner_collect(tobj._meta['crits_type'],
str(tobj.id),
sources,
depth)
def insert_analysis_results(task):
"""
Insert analysis results for this task.
"""
obj_class = class_from_type(task.obj._meta['crits_type'])
ear = EmbeddedAnalysisResult()
tdict = task.to_dict()
tdict['analysis_id'] = tdict['id']
del tdict['id']
ear.merge(arg_dict=tdict)
obj_class.objects(id=task.obj.id).update_one(push__analysis=ear)
def refresh_services(request, crits_type, identifier):
"""
Refresh the Analysis tab with the latest information.
"""
response = {}
# Verify user can see results.
sources = user_sources(request.user.username)
klass = class_from_type(crits_type)
if not klass:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), mimetype="application/json")
if hasattr(klass, 'source'):
obj = klass.objects(id=identifier, source__name__in=sources).first()
else:
obj = klass.objects(id=identifier).first()
if not obj:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), mimetype="application/json")
# Get analysis results.
results = AnalysisResult.objects(object_type=crits_type,
object_id=identifier)
relationship = {
'type': crits_type,
'value': identifier,
'url_key': obj.get_url_key()
}
subscription = {'type': crits_type, 'id': identifier}
service_list = get_supported_services(crits_type)
response['success'] = True
response['html'] = render_to_string(
"services_analysis_listing.html", {
'relationship': relationship,
'subscription': subscription,
'service_results': results,
'crits_type': crits_type,
'identifier': identifier,
'service_list': service_list
}, RequestContext(request))
return HttpResponse(json.dumps(response), mimetype="application/json")
def execute_yargen(relatedSamples, user):
scount = 0
yargen_array = {}
critsMessage = ""
#critsMessage = str(relatedSamples)
#response = {"success": True, "message": critsMessage}
#return response
message = {}
for key, val in relatedSamples.iteritems():
newkey = key.replace(']','')
newkeylist = newkey.split('[')
keyCheck = message.get(int(newkeylist[1]), 'none')
if keyCheck=='none':
message[int(newkeylist[1])]={}
message[int(newkeylist[1])][str(newkeylist[2])]=str(val)
#critsMessage = str(message)
#response = {"success": True, "message": critsMessage}
#return response
for sample in message:
#critsMessage+="\r\nsample [id] - "
#critsMessage+=str(message[sample]['id'])
yargen_array[scount]={}
#yargen_array[scount]['id'] = sample.id
yargen_array[scount]['id'] = message[sample]['id']
#yargen_array[scount]['filename'] = sample.label
yargen_array[scount]['filename'] = message[sample]['label']
#klass = class_from_id(sample.type, sample.id)
#klass = class_from_id(message[sample]['type'], message[sample]['id'])
klass = class_from_type(message[sample]['type'])
#obj = klass.objects(id=sample.id).first()
obj = klass.objects(id=message[sample]['id']).first()
filedata = obj.filedata.read()
yargen_array[scount]['filedata'] = filedata
#critsMessage+="-----FILE------\r\n"
#critsMessage+=filedata
#critsMessage+="\r\n-------END FILE------\r\n"
yargen_array[scount]['size'] = getattr(obj, 'size', '')
scount += 1
import yarGen
critsMessage = yarGen.runMain(yargen_array, critsMessage)
response = {"success": True, "message": critsMessage}
return response
def update_dependency(type_, id_, dep, user, append=False, **kwargs):
"""
Change the dependencies needed for a signature
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: The ObjectId to search for.
:type id_: str
:param data_type_dependency: The new list of dependencies
:type data_type_dependency: list
:param user: The user setting the dependency.
:type user: str
:param append: Should be appended to dependency list?
:type append: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
# Have to unescape the submitted data. Use unescape() to escape
# < and friends. Use urllib2.unquote() to escape %3C and friends.
h = HTMLParser.HTMLParser()
data_type_dependency = h.unescape(dep)
try:
deps = data_type_dependency.split(',')
if append is False:
del obj.data_type_dependency[:]
for item in deps:
item = item.strip()
item = str(item)
if item:
add_new_signature_dependency(item, user)
obj.data_type_dependency.append(item)
obj.save(username=user)
return {'success': True, 'message': "Data type dependency set."}
except ValidationError, e:
return {'success': False, 'message': e}
def refresh_services(request, crits_type, identifier):
"""
Refresh the Analysis tab with the latest information.
"""
response = {}
request.user._setup()
# Verify user can see results.
sources = request.user.get_sources_list()
klass = class_from_type(crits_type)
if not klass:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), content_type="application/json")
if hasattr(klass, 'source'):
obj = klass.objects(id=identifier,source__name__in=sources).first()
else:
obj = klass.objects(id=identifier).first()
if not obj:
msg = 'Could not find object to refresh!'
response['success'] = False
response['html'] = msg
return HttpResponse(json.dumps(response), content_type="application/json")
# Get analysis results.
results = AnalysisResult.objects(object_type=crits_type,
object_id=identifier)
relationship = {'type': crits_type,
'value': identifier}
subscription = {'type': crits_type,
'id': identifier}
service_list = get_supported_services(crits_type)
response['success'] = True
response['html'] = render_to_string("services_analysis_listing.html",
{'relationship': relationship,
'subscription': subscription,
'service_results': results,
'crits_type': crits_type,
'identifier': identifier,
'service_list': service_list},
request=request)
return HttpResponse(json.dumps(response), content_type="application/json")
def add_log(object_type, object_id, analysis_id, log_message, level, analyst):
"""
Add a log entry to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param log_message: The log entry to append.
:type log_message: dict
:param level: The log level.
:type level: str
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
results['message'] = "Could not find object to add log to."
return results
found = False
c = 0
for a in obj.analysis:
if str(a.analysis_id) == analysis_id:
found = True
break
c += 1
if not found:
results['message'] = "Could not find an analysis task to update."
return results
le = EmbeddedAnalysisResult.EmbeddedAnalysisResultLog()
le.message = log_message
le.level = level
le.datetime = str(datetime.datetime.now())
klass.objects(
id=object_id,
analysis__id=analysis_id).update_one(push__analysis__S__log=le)
results['success'] = True
return results
def add_log(object_type, object_id, analysis_id, log_message, level, analyst):
"""
Add a log entry to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param log_message: The log entry to append.
:type log_message: dict
:param level: The log level.
:type level: str
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
klass = class_from_type(object_type)
sources = user_sources(analyst)
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
results['message'] = "Could not find object to add log to."
return results
found = False
c = 0
for a in obj.analysis:
if str(a.analysis_id) == analysis_id:
found = True
break
c += 1
if not found:
results['message'] = "Could not find an analysis task to update."
return results
le = EmbeddedAnalysisResult.EmbeddedAnalysisResultLog()
le.message = log_message
le.level = level
le.datetime = str(datetime.datetime.now())
klass.objects(id=object_id,
analysis__id=analysis_id).update_one(push__analysis__S__log=le)
results['success'] = True
return results
def run(self, argv):
parser = OptionParser()
parser.add_option('-s', '--services', dest='services', help='Service list')
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
default=False,
help='Verbose mode')
parser.add_option('-f', '--filter', dest='query_filter',
help='Query filter')
parser.add_option('-T', '--type', dest='type_', default='Sample',
help='CRITs type query for (default: Sample)')
parser.add_option('-i', '--identifier', dest='identifier',
help='Identifier for type (NOT OBJECT ID)')
parser.add_option('-c', '--config', dest='config', default={},
help='Service configuration')
(opts, args) = parser.parse_args(argv)
service_list = []
if opts.services:
service_list = opts.services.split(',')
if opts.verbose:
self.print_running_services(service_list)
if opts.query_filter:
query = ast.literal_eval(opts.query_filter)
klass = class_from_type(opts.type_)
if not klass:
print "[-] Invalid type."
obj_list = klass.objects(__raw__=query)
if opts.verbose:
self.print_object_stats(obj_list, opts.query_filter)
if opts.identifier:
obj = class_from_value(opts.type_, opts.identifier)
if not obj:
print "[-] Unable to find object."
return
obj_list = [obj]
if opts.verbose:
self.print_object_stats(obj_list)
config = {}
if opts.config:
config = ast.literal_eval(opts.config)
if obj_list and service_list:
self.run_services(service_list, obj_list, opts.verbose, config)
def _insert_analysis_results(self, task):
"""
Insert analysis results for this task.
"""
obj_class = class_from_type(task.context.crits_type)
query = self.get_db_query(task.context)
ear = EmbeddedAnalysisResult()
tdict = task.to_dict()
tdict['analysis_type'] = tdict['type']
tdict['analysis_id'] = tdict['id']
del tdict['type']
del tdict['id']
ear.merge(arg_dict=tdict)
ear.config = AnalysisConfig(**tdict['config'])
obj_class.objects(__raw__=query).update_one(push__analysis=ear)
def _insert_analysis_results(self, task):
"""
Insert analysis results for this task.
"""
obj_class = class_from_type(task.context.crits_type)
query = self.get_db_query(task.context)
ear = EmbeddedAnalysisResult()
tdict = task.to_dict()
tdict['analysis_type'] = tdict['type']
tdict['analysis_id'] = tdict['id']
del tdict['type']
del tdict['id']
ear.merge(arg_dict=tdict)
ear.config = AnalysisConfig(**tdict['config'])
obj_class.objects(__raw__=query).update_one(push__analysis=ear)
def delete_object_file(value):
"""
In the event this is a file (but not PCAP), clean up after ourselves when
deleting an object.
:param value: The value of the object we are deleting.
:type value: str
"""
if not re.match(r"^[a-f\d]{32}$", value, re.I):
return
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
obj_list = (
'Actor',
'Backdoor',
'Campaign',
'Certificate',
'Domain',
'Email',
'Event',
'Exploit',
'Indicator',
'IP',
'PCAP',
'RawData',
'Sample',
'Target',
)
# In order to make sure this object isn't tied to more than one top-level
# object, we need to check the rest of the database. We will at least find
# one instance, which is the one we are going to be removing. If we find
# another instance, then we should not remove the object from GridFS.
count = 0
query = {'objects.value': value}
for obj in obj_list:
obj_class = class_from_type(obj)
count += len(obj_class.objects(__raw__=query))
if count > 1:
break
else:
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
grid.remove({'md5': value})
return
def get_actor_tags_by_type(tag_type):
"""
Get Actor tags based on type. These are tags that could be used for
attribution.
:param tag_type: The type of tags to get.
:type tag_type: str
:return: list
"""
tags = []
if tag_type in ('ActorIntendedEffect', 'ActorMotivation',
'ActorSophistication', 'ActorThreatType'):
obj = class_from_type(tag_type)
results = obj.objects()
tags = [t.name for t in results]
return tags
def finish_task(object_type, object_id, analysis_id, status, analyst):
"""
Finish a task by setting its status to "completed" and setting the finish
date.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param status: The status of the task.
:type status: str ("error", "completed")
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not status:
status = "completed"
if status not in ('error', 'completed'):
status = "completed"
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
params = {'id': object_id}
if hasattr(klass, 'source'):
params['source__name__in'] = user_sources(analyst)
obj = klass.objects(**params).first()
if not obj:
results['message'] = "Could not find object to add results to."
return results
# Update analysis log
date = str(datetime.datetime.now())
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(set__status=status,
set__finish_date=date)
results['success'] = True
return results
def finish_task(object_type, object_id, analysis_id, status, analyst):
"""
Finish a task by setting its status to "completed" and setting the finish
date.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param status: The status of the task.
:type status: str ("error", "completed")
:param analyst: The user updating the log.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not status:
status = "completed"
if status not in ('error', 'completed'):
status = "completed"
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
params = {'id': object_id}
if hasattr(klass, 'source'):
params['source__name__in'] = user_sources(analyst)
obj = klass.objects(**params).first()
if not obj:
results['message'] = "Could not find object to add results to."
return results
# Update analysis log
date = str(datetime.datetime.now())
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(set__status=status,
set__finish_date=date)
results['success'] = True
return results
def delete_object_file(value):
"""
In the event this is a file (but not PCAP), clean up after ourselves when
deleting an object.
:param value: The value of the object we are deleting.
:type value: str
"""
if not re.match(r"^[a-f\d]{32}$", value, re.I):
return
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
obj_list = ('Actor',
'Backdoor',
'Campaign',
'Certificate',
'Domain',
'Email',
'Event',
'Exploit',
'Indicator',
'IP',
'PCAP',
'RawData',
'Sample',
'Target',
)
# In order to make sure this object isn't tied to more than one top-level
# object, we need to check the rest of the database. We will at least find
# one instance, which is the one we are going to be removing. If we find
# another instance, then we should not remove the object from GridFS.
count = 0
query = {'objects.value': value}
for obj in obj_list:
obj_class = class_from_type(obj)
count += len(obj_class.objects(__raw__=query))
if count > 1:
break
else:
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
grid.remove({'md5': value})
return
def handle(self, *args, **options):
"""
Script Execution.
"""
buckets = {}
types = [
"Actor",
"Campaign",
"Certificate",
"Domain",
"Email",
"Event",
"Indicator",
"IP",
"PCAP",
"RawData",
"Signature",
"Sample",
"Target",
]
for otype in types:
klass = class_from_type(otype)
if not klass:
continue
objs = klass.objects().only("bucket_list")
for obj in objs:
for bucket in obj.bucket_list:
if not bucket:
continue # Avoid empty strings
if bucket not in buckets:
buckets[bucket] = Bucket()
buckets[bucket].name = bucket
setattr(buckets[bucket], otype, 1)
else:
buckets[bucket][otype] += 1
# Drop all existing buckets
Bucket.objects().delete()
for bucket in buckets.values():
bucket.save()
def add_log(object_type, object_id, analysis_id, log_message, level, user):
"""
Add a log entry to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param log_message: The log entry to append.
:type log_message: dict
:param level: The log level.
:type level: str
:param user: The user updating the log.
:type user: :class:`crits.core.user.CRITsUser`
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user.get_sources_list()
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
results['message'] = "Could not find object to add results to."
return results
# Update analysis log
le = EmbeddedAnalysisResultLog()
le.message = log_message
le.level = level
le.datetime = str(datetime.datetime.now())
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(push__log=le)
results['success'] = True
else:
results['message'] = "Could not find task to add log to."
return results
def add_log(object_type, object_id, analysis_id, log_message, level, user):
"""
Add a log entry to an analysis task.
:param object_type: The top-level object type.
:type object_type: str
:param object_id: The ObjectId to search for.
:type object_id: str
:param analysis_id: The ID of the task to update.
:type analysis_id: str
:param log_message: The log entry to append.
:type log_message: dict
:param level: The log level.
:type level: str
:param user: The user updating the log.
:type user: :class:`crits.core.user.CRITsUser`
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
results = {'success': False}
if not object_type or not object_id or not analysis_id:
results['message'] = "Must supply object id/type and analysis id."
return results
# Validate user can add service results to this TLO.
klass = class_from_type(object_type)
sources = user.get_sources_list()
obj = klass.objects(id=object_id, source__name__in=sources).first()
if not obj:
results['message'] = "Could not find object to add results to."
return results
# Update analysis log
le = EmbeddedAnalysisResultLog()
le.message = log_message
le.level = level
le.datetime = str(datetime.datetime.now())
ar = AnalysisResult.objects(analysis_id=analysis_id).first()
if ar:
AnalysisResult.objects(id=ar.id).update_one(push__log=le)
results['success'] = True
else:
results['message'] = "Could not find task to add log to."
return results
def get_user_allowed_comments(comments, sources):
"""
Limit the comments to those a user should have access to see.
:param comments: The list of comments.
:type comments: list
:param sources: The sources the user has access to.
:type sources: list
:returns: list of :class:`crits.comments.comment.Comment`
"""
docs = {'Actor': {},
'Campaign':{},
'Certificate':{},
'Domain':{},
'Email':{},
'Event':{},
'Indicator':{},
'IP':{},
'PCAP':{},
'RawData':{},
'Sample':{},
'Target':{}}
for c in comments:
c.comment_to_html()
try:
docs[c.obj_type][c.obj_id].append(c)
except KeyError:
docs[c.obj_type][c.obj_id] = [c]
final_comments = []
for key, val in docs.items():
cls = class_from_type(key)
obj_ids = [v for v in val] #get keys
query = {'_id': {'$in':obj_ids},
'$or': [{'source.name': {'$in':sources}},
{'source': {'$exists': 0}}
]
}
result = cls.objects(__raw__=query).only('id')
for r in result:
final_comments += val[r.id]
final_comments.sort(key=lambda x: x.created, reverse=True)
return final_comments
def get_actor_tags_by_type(tag_type):
"""
Get Actor tags based on type. These are tags that could be used for
attribution.
:param tag_type: The type of tags to get.
:type tag_type: str
:return: list
"""
tags = []
if tag_type in ('ActorIntendedEffect',
'ActorMotivation',
'ActorSophistication',
'ActorThreatType'):
obj = class_from_type(tag_type)
results = obj.objects()
tags = [t.name for t in results]
return tags
def get_user_allowed_comments(comments, sources):
"""
Limit the comments to those a user should have access to see.
:param comments: The list of comments.
:type comments: list
:param sources: The sources the user has access to.
:type sources: list
:returns: list of :class:`crits.comments.comment.Comment`
"""
docs = {
"Actor": {},
"Campaign": {},
"Certificate": {},
"Domain": {},
"Email": {},
"Event": {},
"Indicator": {},
"IP": {},
"PCAP": {},
"RawData": {},
"Sample": {},
"Target": {},
}
for c in comments:
c.comment_to_html()
try:
docs[c.obj_type][c.obj_id].append(c)
except KeyError:
docs[c.obj_type][c.obj_id] = [c]
final_comments = []
for key, val in docs.items():
cls = class_from_type(key)
obj_ids = [v for v in val] # get keys
query = {"_id": {"$in": obj_ids}, "$or": [{"source.name": {"$in": sources}}, {"source": {"$exists": 0}}]}
result = cls.objects(__raw__=query).only("id")
for r in result:
final_comments += val[r.id]
final_comments.sort(key=lambda x: x.created, reverse=True)
return final_comments
def update_signature_type(type_, id_, data_type, user, **kwargs):
"""
Update the Signature data type.
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: ObjectId of the Signature to update.
:type id_: str
:param data_type: The data type to set.
:type data_type: str
:param user: The user updating the data type.
:type user: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
klass = class_from_type(type_)
if not klass:
return {"success": False, "message": "Could not find object."}
if hasattr(klass, "source"):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {"success": False, "message": "Could not find object."}
signature = Signature.objects(id=id_).first()
data_type = SignatureType.objects(name=data_type).first()
if not data_type:
return None
else:
signature.data_type = data_type.name
try:
signature.save(username=user)
return {"success": True}
except ValidationError, e:
return {"success": False, "message": str(e)}
def update_signature_type(type_, id_, data_type, user, **kwargs):
"""
Update the Signature data type.
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: ObjectId of the Signature to update.
:type id_: str
:param data_type: The data type to set.
:type data_type: str
:param user: The user updating the data type.
:type user: str
:returns: dict with keys "success" (boolean) and "message" (str) if failed.
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
signature = Signature.objects(id=id_).first()
data_type = SignatureType.objects(name=data_type).first()
if not data_type:
return None
else:
signature.data_type = data_type.name
try:
signature.save(username=user.username)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': str(e)}
def _update_analysis_results(self, task):
"""
Update analysis results for this task.
"""
# If the task does not currently exist for the given sample in the
# database, add it.
obj_class = class_from_type(task.context.crits_type)
query = self.get_db_query(task.context)
obj = obj_class.objects(__raw__=query).first()
obj_id = obj.id
found = False
c = 0
for a in obj.analysis:
if str(a.analysis_id) == task.task_id:
found = True
break
c += 1
if not found:
logger.warning("Tried to update a task that didn't exist.")
self._insert_analysis_results(task)
else:
# Otherwise, update it.
ear = EmbeddedAnalysisResult()
tdict = task.to_dict()
tdict['analysis_type'] = tdict['type']
tdict['analysis_id'] = tdict['id']
del tdict['type']
del tdict['id']
ear.merge(arg_dict=tdict)
ear.config = AnalysisConfig(**tdict['config'])
obj_class.objects(
id=obj_id,
analysis__id=task.task_id).update_one(set__analysis__S=ear)
def update_title(type_, id_, title, user, **kwargs):
"""
Change signature data for the current version
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: The ObjectId to search for.
:type id_: str
:param title: The new signature title to use.
:type title: str
:param user: The user setting the data value.
:type user: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
# Have to unescape the submitted data. Use unescape() to escape
# < and friends. Use urllib2.unquote() to escape %3C and friends.
h = HTMLParser.HTMLParser()
data = h.unescape(title)
try:
obj.title = data
obj.save(username=title)
return {'success': True, 'message': "Signature title updated."}
except ValidationError, e:
return {'success': False, 'message': e}
def update_max_version(type_, id_, data_type_max_version, user, **kwargs):
"""
Change the max version of the data tool
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: The ObjectId to search for.
:type id_: str
:param data_type_max_version: The new max version to use.
:type data_type_max_version: str
:param user: The user setting the description.
:type user: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(user)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
# Have to unescape the submitted data. Use unescape() to escape
# < and friends. Use urllib2.unquote() to escape %3C and friends.
h = HTMLParser.HTMLParser()
data_type_max_version = h.unescape(data_type_max_version)
try:
obj.data_type_max_version = data_type_max_version
obj.save(username=user.username)
return {'success': True, 'message': "Data type max version set."}
except ValidationError, e:
return {'success': False, 'message': e}
def update_max_version(type_, id_, data_type_max_version, analyst):
"""
Change the max version of the data tool
:param type_: The CRITs type of the top-level object.
:type type_: str
:param id_: The ObjectId to search for.
:type id_: str
:param data_type_max_version: The new max version to use.
:type data_type_max_version: str
:param analyst: The user setting the description.
:type analyst: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
klass = class_from_type(type_)
if not klass:
return {'success': False, 'message': 'Could not find object.'}
if hasattr(klass, 'source'):
sources = user_sources(analyst)
obj = klass.objects(id=id_, source__name__in=sources).first()
else:
obj = klass.objects(id=id_).first()
if not obj:
return {'success': False, 'message': 'Could not find object.'}
# Have to unescape the submitted data. Use unescape() to escape
# < and friends. Use urllib2.unquote() to escape %3C and friends.
h = HTMLParser.HTMLParser()
data_type_max_version = h.unescape(data_type_max_version)
try:
obj.data_type_max_version = data_type_max_version
obj.save(username=analyst)
return {'success': True, 'message': "Data type max version set."}
except ValidationError, e:
return {'success': False, 'message': e}
def inner_collect(obj_type, obj_id, sources, depth):
# Don't keep going if we've already processed this object
if obj_id in objects:
return
klass = class_from_type(obj_type)
if not klass:
return
if hasattr(klass, 'source'):
obj = klass.objects(id=obj_id, source__name__in=sources).first()
else:
obj = klass.objects(id=obj_id).first()
if not obj:
return
objects[obj_id] = obj
if depth == 0:
return
depth -= 1
for r in obj.relationships:
inner_collect(r.rel_type, str(r.object_id), sources, depth)
def generate_timeline(obj_type, obj_id, user):
users_sources = user_sources(user)
obj_class = class_from_type(obj_type)
if hasattr(obj_class, 'source'):
main_obj = obj_class.objects(id=obj_id,
source__name__in=users_sources).first()
else:
main_obj = obj_class.objects(id=obj_id).first()
if not main_obj:
return {'success': False, 'message': 'No starting object found.'}
# timeline is a dictionary.
# the key is the date with no time allowing us to collect a day's events.
# the value is a list of tuples.
# the first item in the tuple should be a datetime string for the event.
# the second element should be a description of the event that happened.
timeline = {}
# creation time
i = "<b>%s</b> was created" % obj_type
append_to_timeline(timeline, main_obj.created, i)
# sources
if hasattr(main_obj, 'source'):
for source in main_obj.source:
if source.name in users_sources:
name = source.name
for instance in source.instances:
i = "Source <b>%s</b> provided %s with a method of <b>'%s'</b> \
and a reference of <b>'%s'</b>" % (
name, obj_type, instance.method, instance.reference)
append_to_timeline(timeline, instance.date, i)
# releasability
for release in main_obj.releasability:
if release.name in users_sources:
name = release.name
for instance in release.instances:
i = "Release to <b>%s</b> added." % name
append_to_timeline(timeline, instance.date, i)
# campaigns
for campaign in main_obj.campaign:
name = campaign.name
confidence = campaign.confidence
description = campaign.description
rev = reverse('crits.campaigns.views.campaign_details', args=[
name,
])
link = '<a href="%s">%s</a>' % (rev, name)
i = "Campaign <b>%s</b> added with a confidence of <b>%s</b> and a \
description of '%s'" % (link, confidence, description)
append_to_timeline(timeline, campaign.date, i)
# objects
for obj in main_obj.obj:
name = obj.name
type_ = obj.object_type
if name == type_:
object_type = name
else:
object_type = "%s - %s" % (name, type_)
value = obj.value
rev = '%s?search_type=object&otype=%s&q=%s&force_full=1' \
% (reverse('crits.core.views.global_search_listing'),
"%s - %s" % (type_, name),
urllib.quote(value))
link = '<a href="%s">%s</a>' % (rev, value)
i = "<b>%s</b> object added with a value of :<br />%s" % (object_type,
link)
append_to_timeline(timeline, obj.date, i)
# relationships
for rel in main_obj.relationships:
tobj = class_from_type(rel.rel_type)
if tobj.objects(id=rel.object_id,
source__name__in=users_sources).only('id').first():
rev = reverse('crits.core.views.details',
args=[
rel.rel_type,
str(rel.object_id),
])
link = '<a href="%s">%s</a>' % (rev, rel.rel_type)
i = "<b>%s</b> was added with a relationship of <b>%s</b>." % (
link, rel.relationship)
append_to_timeline(timeline, rel.date, i)
# comments
cobj = class_from_type("Comment")
comments = cobj.objects(obj_type=obj_type,
obj_id=obj_id,
comment_type="comment")
for comment in comments:
i = "<b>%s</b> made a comment: %s" % (comment.analyst, comment.comment)
append_to_timeline(timeline, comment.created, i)
# analysis
for analysis in main_obj.analysis:
analyst = analysis.analyst
service_name = analysis.service_name
version = analysis.version
results = len(analysis.results)
i = "<b>%s</b> ran <b>%s (%s)</b> and got <b>%d</b> results." % (
analyst, service_name, version, results)
append_to_timeline(timeline, analysis.start_date, i)
# tickets
for ticket in main_obj.tickets:
i = "<b>%s</b> added Ticket <b>%s</b>" % (ticket.analyst,
ticket.ticket_number)
append_to_timeline(timeline, ticket.date, i)
# raw data specific timeline entries
if obj_type == "RawData":
# inline comments
for inline in main_obj.inlines:
i = "<b>%s</b> made an inline comment on line <b>%d</b>: %s" % (
inline.analyst, inline.line, inline.comment)
append_to_timeline(timeline, inline.date, i)
# highlights
for highlight in main_obj.highlights:
i = "<b>%s</b> highlighted line <b>%d</b>: %s" % (
highlight.analyst, highlight.line, highlight.comment)
append_to_timeline(timeline, highlight.date, i)
# versions
robj = class_from_type(obj_type)
versions = robj.objects(link_id=main_obj.link_id).only(
'id', 'version', 'created')
for version in versions:
rev = reverse('crits.raw_data.views.raw_data_details',
args=[
str(version.id),
])
link = '<a href="%s">%d</a>' % (rev, version.version)
i = "Version %s was added." % link
append_to_timeline(timeline, version.created, i)
# indicator specific timeline entries
if obj_type == "Indicator":
# actions
for action in main_obj.actions:
i = "<b>%s</b> added action <b>%s</b> to start on <b>%s</b>" \
% (action.analyst,
action.action_type,
action.begin_date)
i += ", set to <b>%s</b>, with a reason of: <b>%s</b>" \
% (action.active,
action.reason)
append_to_timeline(timeline, action.date, i)
# activity
for activity in main_obj.activity:
i = "<b>%s</b> noted Indicator activity from <b>%s</b> to <b>%s</b> \
and said: %s" % (activity.analyst, activity.start_date,
activity.end_date, activity.description)
append_to_timeline(timeline, activity.date, i)
# sort timeline
sorted_timeline = []
keys = timeline.keys()
keys.sort()
for key in keys:
k = timeline[key]
k.sort(key=lambda tup: tup[0])
sorted_timeline.append((key, k))
html = render_to_string('timeline_contents.html',
{'timeline': sorted_timeline})
return {'success': True, 'message': html}
def to_stix(obj, items_to_convert=[], loaded=False, bin_fmt="raw"):
"""
Converts a CRITs object to a STIX document.
The resulting document includes standardized representations
of all related objects noted within items_to_convert.
:param items_to_convert: The list of items to convert to STIX/CybOX
:type items_to_convert: Either a list of CRITs objects OR
a list of {'_type': CRITS_TYPE, '_id': CRITS_ID} dicts
:param loaded: Set to True if you've passed a list of CRITs objects as
the value for items_to_convert, else leave False.
:type loaded: bool
:param bin_fmt: Specifies the format for Sample data encoding.
Options: None (don't include binary data in STIX output),
"raw" (include binary data as is),
"base64" (base64 encode binary data)
:returns: A dict indicating which items mapped to STIX indicators, ['stix_indicators']
which items mapped to STIX observables, ['stix_observables']
which items are included in the resulting STIX doc, ['final_objects']
and the STIX doc itself ['stix_obj'].
"""
from cybox.common import Time, ToolInformationList, ToolInformation
from stix.common import StructuredText, InformationSource
from stix.core import STIXPackage, STIXHeader
from stix.common.identity import Identity
# These lists are used to determine which CRITs objects
# go in which part of the STIX document.
ind_list = ['Indicator']
obs_list = [
'Certificate', 'Domain', 'Email', 'IP', 'PCAP', 'RawData', 'Sample'
]
actor_list = ['Actor']
# Store message
stix_msg = {
'stix_incidents': [],
'stix_indicators': [],
'stix_observables': [],
'stix_actors': [],
'final_objects': []
}
if not loaded: # if we have a list of object metadata, load it before processing
items_to_convert = [
class_from_id(item['_type'], item['_id'])
for item in items_to_convert
]
# add self to the list of items to STIXify
if obj not in items_to_convert:
items_to_convert.append(obj)
# add any email attachments
attachments = []
for obj in items_to_convert:
if obj._meta['crits_type'] == 'Email':
for rel in obj.relationships:
if rel.relationship == RelationshipTypes.CONTAINS:
atch = class_from_id('Sample', rel.object_id)
if atch not in items_to_convert:
attachments.append(atch)
items_to_convert.extend(attachments)
# grab ObjectId of items
refObjs = {key.id: 0 for key in items_to_convert}
relationships = {}
stix = []
from stix.indicator import Indicator as S_Ind
for obj in items_to_convert:
obj_type = obj._meta['crits_type']
if obj_type == class_from_type('Event')._meta['crits_type']:
stx, release = to_stix_incident(obj)
stix_msg['stix_incidents'].append(stx)
elif obj_type in ind_list: # convert to STIX indicators
stx, releas = to_stix_indicator(obj)
stix_msg['stix_indicators'].append(stx)
refObjs[obj.id] = S_Ind(idref=stx.id_)
elif obj_type in obs_list: # convert to CybOX observable
if obj_type == class_from_type('Sample')._meta['crits_type']:
stx, releas = to_cybox_observable(obj, bin_fmt=bin_fmt)
else:
stx, releas = to_cybox_observable(obj)
# wrap in stix Indicator
ind = S_Ind()
for ob in stx:
ind.add_observable(ob)
ind.title = "CRITs %s Top-Level Object" % obj_type
ind.description = ("This is simply a CRITs %s top-level "
"object, not actually an Indicator. "
"The Observable is wrapped in an Indicator"
" to facilitate documentation of the "
"relationship." % obj_type)
ind.confidence = 'None'
stx = ind
stix_msg['stix_indicators'].append(stx)
refObjs[obj.id] = S_Ind(idref=stx.id_)
elif obj_type in actor_list: # convert to STIX actor
stx, releas = to_stix_actor(obj)
stix_msg['stix_actors'].append(stx)
# get relationships from CRITs objects
for rel in obj.relationships:
if rel.object_id in refObjs:
relationships.setdefault(stx.id_, {})
relationships[stx.id_][rel.object_id] = (
rel.relationship, rel.rel_confidence.capitalize(),
rel.rel_type)
stix_msg['final_objects'].append(obj)
stix.append(stx)
# set relationships on STIX objects
for stix_obj in stix:
for rel in relationships.get(stix_obj.id_, {}):
if isinstance(refObjs.get(rel), S_Ind): # if is STIX Indicator
stix_obj.related_indicators.append(refObjs[rel])
rel_meta = relationships.get(stix_obj.id_)[rel]
stix_obj.related_indicators[-1].relationship = rel_meta[0]
stix_obj.related_indicators[-1].confidence = rel_meta[1]
# Add any Email Attachments to CybOX EmailMessage Objects
if isinstance(stix_obj, S_Ind):
if 'EmailMessage' in stix_obj.observable.object_.id_:
if rel_meta[0] == 'Contains' and rel_meta[
2] == 'Sample':
email = stix_obj.observable.object_.properties
email.attachments.append(refObjs[rel].idref)
tool_list = ToolInformationList()
tool = ToolInformation("CRITs", "MITRE")
tool.version = settings.CRITS_VERSION
tool_list.append(tool)
i_s = InformationSource(time=Time(produced_time=datetime.now()),
identity=Identity(name=settings.COMPANY_NAME),
tools=tool_list)
if obj._meta['crits_type'] == "Event":
stix_desc = obj.description()
stix_int = obj.event_type()
stix_title = obj.title()
else:
stix_desc = "STIX from %s" % settings.COMPANY_NAME
stix_int = "Collective Threat Intelligence"
stix_title = "Threat Intelligence Sharing"
header = STIXHeader(information_source=i_s,
description=StructuredText(value=stix_desc),
package_intents=[stix_int],
title=stix_title)
stix_msg['stix_obj'] = STIXPackage(incidents=stix_msg['stix_incidents'],
indicators=stix_msg['stix_indicators'],
threat_actors=stix_msg['stix_actors'],
stix_header=header,
id_=uuid.uuid4())
return stix_msg
