def run(self, argv):
parser = OptionParser()
parser.add_option('-l', '--list', dest='list_services', action='store_true',
default=False,
help='List available services')
parser.add_option('-t', '--triage', dest='triage', action='store_true',
default=False,
help='Run all triage services')
parser.add_option('-e', '--enabled', dest='enabled', action='store_true',
default=False,
help='Run all enabled services')
parser.add_option('-s', '--services', dest='services', help='Service list')
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
default=False,
help='Verbose mode')
parser.add_option('-f', '--filter', dest='sample_filter',
help='Sample query filter')
parser.add_option('-m', '--md5', dest='md5',
help='md5 of sample')
parser.add_option('-F', '--force', dest='force', action='store_true',
default=False,
help='Force run')
(opts, args) = parser.parse_args(argv)
service_list = []
sample_list = []
if opts.list_services:
self.list_available_services()
if (opts.triage or opts.enabled):
service_list = self.get_service_list(opts.triage, opts.enabled)
if opts.verbose:
self.print_running_services(service_list)
elif (opts.services):
if len(opts.services) > 0:
service_list = opts.services.split(',')
if opts.verbose:
self.print_running_services(service_list)
if (opts.sample_filter):
query = ast.literal_eval(opts.sample_filter)
query_results = mongo_find(settings.COL_SAMPLES, query, {'md5': 1})
sample_list = [(sample["md5"], str(sample["_id"])) for sample in query_results]
if opts.verbose:
self.print_sample_stats(sample_list, opts.sample_filter)
if (opts.md5):
# Given an MD5 we have to get the sample ID.
#
# XXX: This should be extended so we can pass an MD5 of a PCAP.
# The entire script also needs to have an option for ID, so we
# can work with other object types that support services.
obj = class_from_value('Sample', opts.md5)
if not obj:
print "[-] Unable to find object."
return
sample_list = [(opts.md5, obj.id)]
if opts.verbose:
self.print_sample_stats(sample_list)
if sample_list and service_list:
self.run_services(service_list, sample_list, opts.verbose, opts.force)
def run(self, argv):
parser = OptionParser()
parser.add_option('-d', '--domain', dest='domain', help='domain')
parser.add_option('-l', '--domain_list', dest='domain_list', help='domain list')
parser.add_option('-i', '--id_list', dest='id_list', help='id list')
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
default=False,
help='Verbose mode')
parser.add_option('-T', '--type', dest='type_', default='Domain',
help='CRITs type query for (default: Domain)')
parser.add_option('--delete', dest='delete', action='store_true',
default=False,
help='Delete Domains')
parser.add_option('--csv', dest='csv', action='store_true',
default=False,
help='Treat file as CSV')
(opts, args) = parser.parse_args(argv)
domain_list = []
if opts.domain_list:
with open(opts.domain_list) as infile:
for line in infile:
if opts.csv:
domain_list = [x.strip() for x in line.split(',')]
else:
domain_list.append(line.strip())
elif opts.domain:
domain_list = opts.domain.split(',')
if opts.verbose:
self.print_delete_objects(domain_list)
error_list = []
obj_list = []
for domain in domain_list:
#domain_obj = Domain.objects(domain_iexact=domain).first()
obj = class_from_value(opts.type_, domain)
if not obj:
error_list.append(domain)
else:
obj_list.append(obj)
if opts.verbose:
self.print_found_objects(obj_list, error_list)
if opts.id_list:
with open(opts.id_list) as infile:
for line in infile:
result = json.loads(line.strip())
obj = class_from_id(opts.type_, result['object_id'])
obj_list.append(obj)
if opts.delete:
self.delete_domains(obj_list, opts.type_, 0.5)
else:
self.run_analysis_cleanup(obj_list, opts.type_, 0.5)
print("Done!")
def relate_objects(self):
"""
for now we are relating all objects to the event with a common
relationship type that is most likely inaccurate. Need to get actual
relationship out of the cybox document once we are storing it there.
"""
for (type_, event_id) in self.events:
event_obj = class_from_value(type_, event_id)
finished_objects = [event_obj]
for (t, v) in self.samples + self.emails + self.indicators:
obj = class_from_value(t, v)
obj.add_relationship(event_obj,
rel_type="Related_To",
analyst=self.source_instance.analyst)
finished_objects.append(obj)
for f in finished_objects:
f.save(username=self.source_instance.analyst)
# If we have no event relate every object to every other object.
if not self.events:
finished_objects = []
for (t, v) in self.samples + self.emails + self.indicators:
obj = class_from_value(t, v)
# Prime the list...
if not finished_objects:
finished_objects.append(obj)
continue
for right in finished_objects:
obj.add_relationship(right,
rel_type="Related_To",
analyst=self.source_instance.analyst)
finished_objects.append(obj)
for f in finished_objects:
f.save(username=self.source_instance.analyst)
def relate_objects(self):
"""
for now we are relating all objects to the event with a common
relationship type that is most likely inaccurate. Need to get actual
relationship out of the cybox document once we are storing it there.
"""
for (type_, event_id) in self.events:
event_obj = class_from_value(type_, event_id)
finished_objects = [event_obj]
for (t, v) in self.samples + self.emails + self.indicators:
obj = class_from_value(t, v)
obj.add_relationship(event_obj,
rel_type="Related_To",
analyst=self.source_instance.analyst)
finished_objects.append(obj)
for f in finished_objects:
f.save(username=self.source_instance.analyst)
# If we have no event relate every object to every other object.
if not self.events:
finished_objects = []
for (t, v) in self.samples + self.emails + self.indicators:
obj = class_from_value(t, v)
# Prime the list...
if not finished_objects:
finished_objects.append(obj)
continue
for right in finished_objects:
obj.add_relationship(right,
rel_type="Related_To",
analyst=self.source_instance.analyst)
finished_objects.append(obj)
for f in finished_objects:
f.save(username=self.source_instance.analyst)
def run(self, argv):
parser = OptionParser()
parser.add_option('-s', '--services', dest='services', help='Service list')
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
default=False,
help='Verbose mode')
parser.add_option('-f', '--filter', dest='query_filter',
help='Query filter')
parser.add_option('-T', '--type', dest='type_', default='Sample',
help='CRITs type query for (default: Sample)')
parser.add_option('-i', '--identifier', dest='identifier',
help='Identifier for type (NOT OBJECT ID)')
parser.add_option('-c', '--config', dest='config', default={},
help='Service configuration')
(opts, args) = parser.parse_args(argv)
service_list = []
if opts.services:
service_list = opts.services.split(',')
if opts.verbose:
self.print_running_services(service_list)
if opts.query_filter:
query = ast.literal_eval(opts.query_filter)
klass = class_from_type(opts.type_)
if not klass:
print "[-] Invalid type."
obj_list = klass.objects(__raw__=query)
if opts.verbose:
self.print_object_stats(obj_list, opts.query_filter)
if opts.identifier:
obj = class_from_value(opts.type_, opts.identifier)
if not obj:
print "[-] Unable to find object."
return
obj_list = [obj]
if opts.verbose:
self.print_object_stats(obj_list)
config = {}
if opts.config:
config = ast.literal_eval(opts.config)
if obj_list and service_list:
self.run_services(service_list, obj_list, opts.verbose, config)
def target_details(request, email_address=None):
"""
Target modification form generation.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param email_address: The email address of the Target to get details for..
:type email_address: str
:returns: :class:`django.http.HttpResponse`
"""
if email_address is None:
form = TargetInfoForm()
else:
target = class_from_value("Target", email_address)
if not target:
form = TargetInfoForm(initial={"email_address": email_address})
else:
form = TargetInfoForm(initial=target.to_dict())
return render_to_response("target_form.html", {"form": form}, RequestContext(request))
def target_details(request, email_address=None):
"""
Target modification form generation.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param email_address: The email address of the Target to get details for..
:type email_address: str
:returns: :class:`django.http.HttpResponse`
"""
if email_address is None:
form = TargetInfoForm()
else:
target = class_from_value('Target', email_address)
if not target:
form = TargetInfoForm(initial={'email_address': email_address})
else:
form = TargetInfoForm(initial=target.to_dict())
return render_to_response('target_form.html', {'form': form},
RequestContext(request))
def handle_cert_file(filename,
data,
source_name,
user=None,
description=None,
related_md5=None,
method='',
reference='',
tlp=None,
relationship=None,
bucket_list=None,
ticket=None,
related_id=None,
related_type=None,
relationship_type=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param reference: A reference to the source of this Certificate.
:type reference: str
:param tlp: The TLP for this certificate.
:type tlp: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
if ((related_type and not (related_id or related_md5))
or (not related_type and (related_id or related_md5))):
status = {
'success':
False,
'message':
'Must specify both related_type and related_id or related_md5.'
}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {'success': False, 'message': 'Related object not found.'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
reference=reference,
method=method,
tlp=tlp,
analyst=user.username)
else:
return {
"success":
False,
"message":
"User does not have permission to add objects \
using source %s." % str(source_name)
}
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name,
method=method,
reference=reference,
tlp=tlp)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s, method=method, reference=reference, tlp=tlp)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if AnalysisResult.objects(object_id=str(cert.id)).count() < 1 and data:
run_triage(cert, user)
# update relationship if a related top-level object is supplied
if related_obj and cert:
if relationship_type:
relationship = RelationshipTypes.inverse(
relationship=relationship_type)
if not relationship:
relationship = RelationshipTypes.RELATED_TO
cert.add_relationship(related_obj,
relationship,
analyst=user,
get_rels=False)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
'id': str(cert.id),
'object': cert
}
return status
def handle_pcap_file(
filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method=None,
relationship=None,
bucket_list=None,
ticket=None,
):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {"success": False, "message": "No data object passed in"}
return status
if len(data) <= 0:
status = {"success": False, "message": "Data length <= 0"}
return status
if (related_type and not (related_id or related_md5)) or (not related_type and (related_id or related_md5)):
status = {"success": False, "message": "Must specify both related_type and related_id or related_md5."}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {"success": False, "message": "Related object not found."}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name, method=method, reference="", analyst=user)
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if not relationship:
relationship = "Related_To"
pcap.add_relationship(rel_item=related_obj, rel_type=relationship, analyst=user, get_rels=False)
related_obj.save(username=user)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(data, pcap, user)
status = {"success": True, "message": "Uploaded pcap", "md5": md5, "id": str(pcap.id)}
return status
def handle_pcap_file(filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method='',
reference='',
relationship=None,
bucket_list=None,
ticket=None):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param reference: A reference to the source of this PCAP.
:type reference: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
if ((related_type and not (related_id or related_md5))
or (not related_type and (related_id or related_md5))):
status = {
'success':
False,
'message':
'Must specify both related_type and related_id or related_md5.'
}
return status
if not source_name:
return {"success": False, "message": "Missing source information."}
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {'success': False, 'message': 'Related object not found.'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference=reference,
analyst=user)
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s, method=method, reference=reference)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if not relationship:
relationship = "Related_To"
pcap.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(pcap, user)
status = {
'success': True,
'message': 'Uploaded pcap',
'md5': md5,
'id': str(pcap.id),
'object': pcap
}
return status
def handle_cert_file(filename, data, source_name, user=None,
description=None, related_id=None, related_md5=None,
related_type=None, method=None, relationship=None,
bucket_list=None, ticket=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_id: ObjectId of a top-level object related to this Certificate.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {
'success': False,
'message': 'No data object passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if ((related_type and not (related_id or related_md5)) or
(not related_type and (related_id or related_md5))):
status = {
'success': False,
'message': 'Must specify both related_type and related_id or related_md5.'
}
return status
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {
'success': False,
'message': 'Related object not found.'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference='',
analyst=user)
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if len(cert.analysis) < 1 and data:
run_triage(cert, user)
# update relationship if a related top-level object is supplied
if related_obj and cert:
if not relationship:
relationship = "Related_To"
cert.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
'id': str(cert.id)
}
return status
def format_object(obj_type, obj_id, data_format="yaml", cleanse=True,
obj_sources=[], remove_source=False, remove_rels=False,
remove_schema_version=False, remove_campaign=False,
remove_buckets=False, remove_releasability=False,
remove_unsupported=False):
"""
Formats a top-level object for utilization in certain conditions. Removes
CRITs-internal necessary data so users editing the document via the
interface don't alter or have the ability to overwrite things they should
not.
:param obj_type: The CRITs type of the top-level object to format.
:type obj_type: str
:param obj_id: The ObjectId to search for.
:type obj_id: str
:param data_format: The format of the returned data.
:type data_format: str of "yaml" or "json"
:param cleanse: Remove "to", "actions", "releasability", and "bucket_list"
if this is an Email or Indicator.
:type cleanse: boolean
:param obj_sources: The sources to overwrite into the document or to set
the source list to an empty list if remove_source is
False.
:type obj_sources: list
:param remove_source: Remove the source key from the document.
:type remove_source: boolean
:param remove_rels: Remove the relationships key from the document.
:type remove_rels: boolean
:param remove_schema_version: Remove the schema_version key from the
document.
:type remove_schema_version: boolean
:param remove_campaign: Remove the campaign key from the document.
:type remove_campaign: boolean
:param remove_buckets: Remove the bucket_list key from the document.
:type remove_buckets: boolean
:param remove_releasability: Remove the releasability key from the document.
:type remove_releasability: boolean
:param remove_unsupported: Remove the unsupported_attrs key from the document.
:type remove_unsupported: boolean
:returns: str
"""
collection = settings.CRITS_TYPES[obj_type]
obj_class = class_from_value(obj_type, obj_id)
if not obj_class:
return ""
data = obj_class.to_dict()
if data is None:
return ""
# Emails use raw_header (singular) as the attribute but store it as
# raw_headers (plural) in the database. When viewing an email in YAML
# or JSON convert from plural to singular. This will allow a copy/paste
# of these views to be imported correctly.
if 'raw_headers' in data:
data['raw_header'] = data['raw_headers']
del data['raw_headers']
if cleanse and collection in [settings.COL_EMAIL, settings.COL_INDICATORS]:
if "to" in data:
del data["to"]
if "actions" in data:
del data["actions"]
if "releasability" in data:
del data["releasability"]
if "bucket_list" in data:
del data["bucket_list"]
if remove_source and 'source' in data:
del data["source"]
elif 'source' in data:
data['source'] = obj_sources
if remove_rels and 'relationships' in data:
del data["relationships"]
if remove_rels and 'objects' in data:
del data["objects"]
if remove_schema_version and 'schema_version' in data:
del data["schema_version"]
if remove_campaign and 'campaign' in data:
del data["campaign"]
del data["_id"]
if data.has_key("modified"):
del data["modified"]
if remove_buckets and 'bucket_list' in data:
del data['bucket_list']
if remove_releasability and 'releasability' in data:
del data['releasability']
if remove_unsupported and 'unsupported_attrs' in data:
del data['unsupported_attrs']
data = json.dumps(convert_datetimes_to_string(data),
default=json_util.default)
if data_format == "yaml":
data = yaml.dump(yaml.load(data), default_flow_style=False)
elif data_format == "json":
data = json.dumps(json.loads(data))
return data
def handle_pcap_file(filename, data, source_name, user=None,
description=None, related_id=None, related_md5=None,
related_type=None, method='', reference='', tlp='',
relationship=None, bucket_list=None, ticket=None):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param reference: A reference to the source of this PCAP.
:type reference: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {
'success': False,
'message': 'No data object passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if ((related_id or related_md5) and not related_type):
status = {
'success': False,
'message': 'Must specify both related_type and related_id or related_md5.'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {
'success': False,
'message': 'Related object not found.'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
if user.check_source_write(source_name):
s = create_embedded_source(source_name,
method=method,
reference=reference,
tlp=tlp,
analyst=user.username)
else:
return {"success":False,
"message": "User does not have permission to add object \
using source %s." % source_name}
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s, method=method, reference=reference)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if relationship:
relationship=RelationshipTypes.inverse(relationship=relationship)
else:
relationship = RelationshipTypes.RELATED_TO
pcap.add_relationship(related_obj,
relationship,
analyst=user,
get_rels=False)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(pcap, user)
status = {
'success': True,
'message': 'Uploaded pcap',
'md5': md5,
'id': str(pcap.id),
'object': pcap
}
return status
def run(self, argv):
parser = OptionParser()
parser.add_option('-l',
'--list',
dest='list_services',
action='store_true',
default=False,
help='List available services')
parser.add_option('-t',
'--triage',
dest='triage',
action='store_true',
default=False,
help='Run all triage services')
parser.add_option('-e',
'--enabled',
dest='enabled',
action='store_true',
default=False,
help='Run all enabled services')
parser.add_option('-s',
'--services',
dest='services',
help='Service list')
parser.add_option('-v',
'--verbose',
dest='verbose',
action='store_true',
default=False,
help='Verbose mode')
parser.add_option('-f',
'--filter',
dest='sample_filter',
help='Sample query filter')
parser.add_option('-m', '--md5', dest='md5', help='md5 of sample')
parser.add_option('-F',
'--force',
dest='force',
action='store_true',
default=False,
help='Force run')
(opts, args) = parser.parse_args(argv)
service_list = []
sample_list = []
if opts.list_services:
self.list_available_services()
if (opts.triage or opts.enabled):
service_list = self.get_service_list(opts.triage, opts.enabled)
if opts.verbose:
self.print_running_services(service_list)
elif (opts.services):
if len(opts.services) > 0:
service_list = opts.services.split(',')
if opts.verbose:
self.print_running_services(service_list)
if (opts.sample_filter):
query = ast.literal_eval(opts.sample_filter)
query_results = mongo_find(settings.COL_SAMPLES, query, {'md5': 1})
sample_list = [(sample["md5"], str(sample["_id"]))
for sample in query_results]
if opts.verbose:
self.print_sample_stats(sample_list, opts.sample_filter)
if (opts.md5):
# Given an MD5 we have to get the sample ID.
#
# XXX: This should be extended so we can pass an MD5 of a PCAP.
# The entire script also needs to have an option for ID, so we
# can work with other object types that support services.
obj = class_from_value('Sample', opts.md5)
if not obj:
print "[-] Unable to find object."
return
sample_list = [(opts.md5, obj.id)]
if opts.verbose:
self.print_sample_stats(sample_list)
if sample_list and service_list:
self.run_services(service_list, sample_list, opts.verbose,
opts.force)
def format_object(obj_type,
obj_id,
data_format="yaml",
cleanse=True,
obj_sources=[],
remove_source=False,
remove_rels=False,
remove_schema_version=False,
remove_campaign=False,
remove_buckets=False,
remove_releasability=False,
remove_unsupported=False):
"""
Formats a top-level object for utilization in certain conditions. Removes
CRITs-internal necessary data so users editing the document via the
interface don't alter or have the ability to overwrite things they should
not.
:param obj_type: The CRITs type of the top-level object to format.
:type obj_type: str
:param obj_id: The ObjectId to search for.
:type obj_id: str
:param data_format: The format of the returned data.
:type data_format: str of "yaml" or "json"
:param cleanse: Remove "to", "actions", "releasability", and "bucket_list"
if this is an Email or Indicator.
:type cleanse: boolean
:param obj_sources: The sources to overwrite into the document or to set
the source list to an empty list if remove_source is
False.
:type obj_sources: list
:param remove_source: Remove the source key from the document.
:type remove_source: boolean
:param remove_rels: Remove the relationships key from the document.
:type remove_rels: boolean
:param remove_schema_version: Remove the schema_version key from the
document.
:type remove_schema_version: boolean
:param remove_campaign: Remove the campaign key from the document.
:type remove_campaign: boolean
:param remove_buckets: Remove the bucket_list key from the document.
:type remove_buckets: boolean
:param remove_releasability: Remove the releasability key from the document.
:type remove_releasability: boolean
:param remove_unsupported: Remove the unsupported_attrs key from the document.
:type remove_unsupported: boolean
:returns: str
"""
collection = settings.CRITS_TYPES[obj_type]
obj_class = class_from_value(obj_type, obj_id)
if not obj_class:
return ""
data = obj_class.to_dict()
if data is None:
return ""
# Emails use raw_header (singular) as the attribute but store it as
# raw_headers (plural) in the database. When viewing an email in YAML
# or JSON convert from plural to singular. This will allow a copy/paste
# of these views to be imported correctly.
if 'raw_headers' in data:
data['raw_header'] = data['raw_headers']
del data['raw_headers']
if cleanse and collection in [settings.COL_EMAIL, settings.COL_INDICATORS]:
if "to" in data:
del data["to"]
if "actions" in data:
del data["actions"]
if "releasability" in data:
del data["releasability"]
if "bucket_list" in data:
del data["bucket_list"]
if remove_source and 'source' in data:
del data["source"]
elif 'source' in data:
data['source'] = obj_sources
if remove_rels and 'relationships' in data:
del data["relationships"]
if remove_rels and 'objects' in data:
del data["objects"]
if remove_schema_version and 'schema_version' in data:
del data["schema_version"]
if remove_campaign and 'campaign' in data:
del data["campaign"]
del data["_id"]
if data.has_key("modified"):
del data["modified"]
if remove_buckets and 'bucket_list' in data:
del data['bucket_list']
if remove_releasability and 'releasability' in data:
del data['releasability']
if remove_unsupported and 'unsupported_attrs' in data:
del data['unsupported_attrs']
data = json.dumps(convert_datetimes_to_string(data),
default=json_util.default)
if data_format == "yaml":
data = yaml.dump(yaml.load(data), default_flow_style=False)
elif data_format == "json":
data = json.dumps(json.loads(data))
return data
def run(self, argv):
parser = OptionParser()
parser.add_option('-s',
'--services',
dest='services',
help='Service list')
parser.add_option('-v',
'--verbose',
dest='verbose',
action='store_true',
default=False,
help='Verbose mode')
parser.add_option('-f',
'--filter',
dest='query_filter',
help='Query filter')
parser.add_option('-T',
'--type',
dest='type_',
default='Sample',
help='CRITs type query for (default: Sample)')
parser.add_option('-i',
'--identifier',
dest='identifier',
help='Identifier for type (NOT OBJECT ID)')
parser.add_option('-c',
'--config',
dest='config',
default={},
help='Service configuration')
(opts, args) = parser.parse_args(argv)
service_list = []
if opts.services:
service_list = opts.services.split(',')
if opts.verbose:
self.print_running_services(service_list)
if opts.query_filter:
query = ast.literal_eval(opts.query_filter)
klass = class_from_type(opts.type_)
if not klass:
print "[-] Invalid type."
obj_list = klass.objects(__raw__=query)
if opts.verbose:
self.print_object_stats(obj_list, opts.query_filter)
if opts.identifier:
obj = class_from_value(opts.type_, opts.identifier)
if not obj:
print "[-] Unable to find object."
return
obj_list = [obj]
if opts.verbose:
self.print_object_stats(obj_list)
config = {}
if opts.config:
config = ast.literal_eval(opts.config)
if obj_list and service_list:
self.run_services(service_list, obj_list, opts.verbose, config)
def handle_cert_file(filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method=None,
relationship=None,
bucket_list=None,
ticket=None):
"""
Add a Certificate.
:param filename: The filename of the Certificate.
:type filename: str
:param data: The filedata of the Certificate.
:type data: str
:param source_name: The source which provided this Certificate.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the Certificate.
:type user: str
:param description: Description of the Certificate.
:type description: str
:param related_id: ObjectId of a top-level object related to this Certificate.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this Certificate.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this Certificate.
:type method: str
:param relationship: The relationship between the parent and the Certificate.
:type relationship: str
:param bucket_list: Bucket(s) to add to this Certificate
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this Certificate
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate Certificate
cert = Certificate.objects(md5=md5).first()
if not cert:
cert = Certificate()
cert.filename = filename
cert.created = timestamp
cert.size = len(data)
cert.description = description
cert.md5 = md5
# generate source information and add to certificate
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference='',
analyst=user)
cert.add_source(s)
elif isinstance(source_name, EmbeddedSource):
cert.add_source(source_name)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
cert.add_source(s)
if bucket_list:
cert.add_bucket_list(bucket_list, user)
if ticket:
cert.add_ticket(ticket, user)
# add file to GridFS
if not isinstance(cert.filedata.grid_id, ObjectId):
cert.add_file_data(data)
# save cert
cert.save(username=user)
cert.reload()
# run certificate triage
if len(cert.analysis) < 1 and data:
run_triage(data, cert, user)
# update relationship if a related top-level object is supplied
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if related_obj and cert:
if not relationship:
relationship = "Related_To"
cert.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
cert.save(username=user)
status = {
'success': True,
'message': 'Uploaded certificate',
'md5': md5,
}
return status
def forge_relationship(type_=None, id_=None,
class_=None, right_type=None,
right_id=None, right_class=None,
rel_type=None, rel_date=None,
user=None, rel_reason="N/A",
rel_confidence='unknown', get_rels=False, **kwargs):
"""
Forge a relationship between two top-level objects.
:param type_: The type of first top-level object to relate to.
:type type_: str
:param id_: The ObjectId of the first top-level object.
:type id_: str
:param class_: The first top-level object to relate to.
:type class_: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param right_type: The type of second top-level object to relate to.
:type right_type: str
:param right_id: The ObjectId of the second top-level object.
:type right_id: str
:param right_class: The second top-level object to relate to.
:type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param rel_type: The type of relationship.
:type rel_type: str
:param rel_date: The date this relationship applies.
:type rel_date: datetime.datetime
:param user: The user forging this relationship.
:type user: str
:param rel_reason: The reason for the relationship.
:type rel_reason: str
:param rel_confidence: The confidence of the relationship.
:type rel_confidence: str
:param get_rels: Return the relationships after forging.
:type get_rels: boolean
:returns: dict with keys "success" (boolean) and "message" (str if
failed, dict if successful)
"""
if rel_date is None or rel_date == 'None':
rel_date = None
elif isinstance(rel_date, basestring) and rel_date != '':
rel_date = parse(rel_date, fuzzy=True)
elif not isinstance(rel_date, datetime.datetime):
rel_date = None
if not class_:
if type_ and id_:
class_ = class_from_value(type_, id_)
if not class_:
class_ = class_from_id(type_, id_)
if not class_:
return {'success': False,
'message': "Unable to get left object."}
else:
return {'success': False,
'message': "Need a valid left type and id"}
try:
# forge relationship
if right_class:
results = class_.add_relationship(right_class,
rel_type,
rel_date=rel_date,
analyst=user,
rel_confidence=rel_confidence,
rel_reason=rel_reason)
else:
if right_type and right_id:
if user in 'taxii':
results = class_.set_relationship(right_id,
rel_type, right_type,
rel_date=rel_date,
analyst=user,
rel_confidence=rel_confidence,
rel_reason=rel_reason)
else: #Always check for class from value first
rel_item = class_from_value(right_type, right_id)
if not rel_item:
rel_item = class_from_id(right_type, right_id)
if rel_item:
results = class_.add_relationship(rel_item,
rel_type,
rel_date=rel_date,
analyst=user,
rel_confidence=rel_confidence,
rel_reason=rel_reason)
else:
return {'success': False,
'message': "Failed to get right object"}
else:
return {'success': False,
'message': "Need a valid right type and id"}
except Exception, e:
return {'success': False, 'message': e}
def update_relationship_confidences(left_class=None, right_class=None,
left_type=None, left_id=None,
right_type=None, right_id=None,
rel_type=None, rel_date=None,
new_type=None,analyst=None,
new_confidence='unknown'):
"""
Update the relationship type between two top-level objects.
:param left_class: The first top-level object.
:type left_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param right_class: The second top-level object.
:type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param left_type: The type of first top-level object.
:type left_type: str
:param left_id: The ObjectId of the first top-level object.
:type left_id: str
:param right_type: The type of second top-level object.
:type right_type: str
:param right_id: The ObjectId of the second top-level object.
:type right_id: str
:param rel_type: The type of relationship.
:type rel_type: str
:param rel_date: The date this relationship applies.
:type rel_date: datetime.datetime
:param analyst: The user updating this relationship.
:type analyst: str
:param new_confidence: The new confidence level.
:type new_confidence: str
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if rel_date is None or rel_date == 'None':
rel_date = None
elif isinstance(rel_date, basestring) and rel_date != '':
rel_date = parse(rel_date, fuzzy=True)
elif not isinstance(rel_date, datetime.datetime):
rel_date = None
if not left_class:
if left_type and left_id:
left_class = class_from_value(left_type, left_id)
else:
return {'success': False,
'message': "Need a valid left type and id"}
# update relationship
if right_class:
results = left_class.edit_relationship_confidence(rel_item=right_class,
rel_type=rel_type,
rel_date=rel_date,
new_confidence=new_confidence,
analyst=analyst)
left_class.save(username=analyst)
right_class.save(username=analyst)
else:
if right_type and right_id:
results = left_class.edit_relationship_confidence(type_=right_type,
rel_id=right_id,
rel_type=rel_type,
rel_date=rel_date,
new_confidence=new_confidence,
analyst=analyst)
left_class.save(username=analyst)
else:
return {'success': False,
'message': "Need a valid right type and id"}
return results
def delete_relationship(left_class=None, right_class=None,
left_type=None, left_id=None,
right_type=None, right_id=None,
rel_type=None, rel_date=None,
analyst=None, get_rels=True):
"""
Delete a relationship between two top-level objects.
:param left_class: The first top-level object.
:type left_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param right_class: The second top-level object.
:type right_class: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param left_type: The type of first top-level object.
:type left_type: str
:param left_id: The ObjectId of the first top-level object.
:type left_id: str
:param right_type: The type of second top-level object.
:type right_type: str
:param right_id: The ObjectId of the second top-level object.
:type right_id: str
:param rel_type: The type of relationship.
:type rel_type: str
:param rel_date: The date this relationship applies.
:type rel_date: datetime.datetime
:param analyst: The user deleting this relationship.
:type analyst: str
:param get_rels: Return the relationships after forging.
:type get_rels: boolean
:returns: dict with keys "success" (boolean) and "message" (str if
failed, dict if successful)
"""
if rel_date is None or rel_date == 'None':
rel_date = None
elif isinstance(rel_date, basestring) and rel_date != '':
rel_date = parse(rel_date, fuzzy=True)
elif not isinstance(rel_date, datetime.datetime):
rel_date = None
if not left_class:
if left_type and left_id:
left_class = class_from_value(left_type, left_id)
if not left_class:
return {'success': False,
'message': "Unable to get object."}
else:
return {'success': False,
'message': "Need a valid left type and id"}
# delete relationship
if not right_class:
if right_type and right_id:
right_class = class_from_value(right_type, right_id)
else:
return {'success': False,
'message': "Need a valid right type and id"}
if not right_class:
results = left_class.delete_relationship(rel_type=rel_type,
rel_date=rel_date,
analyst=analyst,
rel_id=right_id,
type_=right_type)
else:
results = left_class.delete_relationship(rel_item=right_class,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst)
if results['success']:
left_class.save(username=analyst)
left_class.reload()
if get_rels:
results['relationships'] = left_class.sort_relationships("%s" % analyst, meta=True)
return results
