def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
def delete_pcap(pcap_md5, username=None):
"""
Delete a PCAP.
:param pcap_md5: The MD5 of the PCAP to delete.
:type pcap_md5: str
:param username: The user deleting the pcap.
:type username: str
:returns: True, False
"""
pcap = PCAP.objects(md5=pcap_md5).first()
if pcap:
pcap.delete(username=username)
return True
else:
return False
def delete_pcap(pcap_md5, username=None):
"""
Delete a PCAP.
:param pcap_md5: The MD5 of the PCAP to delete.
:type pcap_md5: str
:param username: The user deleting the pcap.
:type username: str
:returns: True, False
"""
pcap = PCAP.objects(md5=pcap_md5).first()
if pcap:
pcap.delete(username=username)
return True
else:
return False
def create_pcap_context(self, identifier, username):
# .only() is currently broken in MongoEngine :(
#fields = ('filename', 'length', 'filedata')
#pcap = PCAP.objects(id=identifier).only(*fields).first()
pcap = PCAP.objects(id=identifier).first()
if not pcap:
raise ValueError("PCAP not found in database")
data = pcap.filedata.read()
if not data:
raise ValueError("PCAP not found in GridFS")
pcap_md5 = pcap.md5
self._check_length(data, getattr(pcap, 'length', 0))
return PCAPContext(username, data, pcap_md5, pcap.to_dict())
def create_pcap_context(self, identifier, username):
# .only() is currently broken in MongoEngine :(
#fields = ('filename', 'length', 'filedata')
#pcap = PCAP.objects(id=identifier).only(*fields).first()
pcap = PCAP.objects(id=identifier).first()
if not pcap:
raise ValueError("PCAP not found in database")
data = pcap.filedata.read()
if not data:
raise ValueError("PCAP not found in GridFS")
pcap_md5 = pcap.md5
self._check_length(data, getattr(pcap, 'length', 0))
return PCAPContext(username, data, pcap_md5, pcap.to_dict())
def update_pcap_description(md5, description, analyst):
"""
Update a PCAP description.
:param md5: The MD5 of the PCAP to update.
:type md5: str
:param description: The new description.
:type description: str
:param analyst: The user updating the description.
:type analyst: str
:returns: None, ValidationError
"""
pcap = PCAP.objects(md5=md5).first()
pcap.description = description
try:
pcap.save(username=analyst)
return None
except ValidationError, e:
return e
def update_pcap_description(md5, description, analyst):
"""
Update a PCAP description.
:param md5: The MD5 of the PCAP to update.
:type md5: str
:param description: The new description.
:type description: str
:param analyst: The user updating the description.
:type analyst: str
:returns: None, ValidationError
"""
pcap = PCAP.objects(md5=md5).first()
pcap.description = description
try:
pcap.save(username=analyst)
return None
except ValidationError, e:
return e
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
def handle_pcap_file(filename, data, source_name, user=None,
description=None, related_id=None, related_md5=None,
related_type=None, method=None, reference=None,
relationship=None, bucket_list=None, ticket=None):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param reference: A reference to the source of this PCAP.
:type reference: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {
'success': False,
'message': 'No data object passed in'
}
return status
if len(data) <= 0:
status = {
'success': False,
'message': 'Data length <= 0'
}
return status
if ((related_type and not (related_id or related_md5)) or
(not related_type and (related_id or related_md5))):
status = {
'success': False,
'message': 'Must specify both related_type and related_id or related_md5.'
}
return status
if not source_name:
return {"success" : False, "message" : "Missing source information."}
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {
'success': False,
'message': 'Related object not found.'
}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference=reference,
analyst=user)
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s, method=method, reference=reference)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if not relationship:
relationship = "Related_To"
pcap.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(pcap, user)
status = {
'success': True,
'message': 'Uploaded pcap',
'md5': md5,
'id': str(pcap.id),
}
return status
def chopshop_carver(pcap_md5, options, analyst):
# Make sure we can find ChopShop
sc = get_config('ChopShop')
if not sc:
return {
'success': False,
'message': 'Could not find ChopShop service.'
}
shop_path = "%s/shop" % str(sc['basedir'])
if not os.path.exists(shop_path):
return {
'success': False,
'message': "ChopShop shop path does not exist."
}
sys.path.append(shop_path)
import ChopLib as CL
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}
# Until we have an smtp_extractor in ChopShop we have to resort to
# to (ab)using payloads to dump the entire TCP stream and letting
# handle_eml() process everything. We also use the payloads module
# for handling raw carves. If a user wants to do SMTP and raw
# simultaneously it won't work because we can't distinguish one
# payloads module from another.
if options.get('raw', False) and options.get('smtp', False):
return {
'success': False,
'message': "Can not process SMTP and raw simultaneously."
}
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return {'success': False, 'message': "No PCAP found."}
pcap_data = pcap.filedata.read()
if not pcap_data:
return {
'success': False,
'message': "Could not get PCAP from GridFS: %s" % pcap_md5
}
source = pcap['source'][0]['name'] # XXX: This kind of sucks...
# Create module string to pass to ChopShop
modules = []
if options.get('http_resp', False) or options.get('http_req', False):
modules.append("http | http_extractor")
if options.get('smtp', False) or options.get('raw', False):
# ChopShop really needs an smtp_extractor, but there's no good
# capability to do that yet. Maybe one day I'll build one. :)
# For now, just use payloads and let handle_eml() sort it out.
#
# Raw carving works exactly the same way, just post-processed
# differently.
modules.append("payloads -b")
if not modules:
return {'success': False, 'message': "No modules specified."}
mod_string = ';'.join(mod for mod in modules)
from ChopLib import ChopLib
from ChopUi import ChopUi
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = str(sc['basedir'])
choplib.modules = mod_string
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# Write the pcap data to a temporary file.
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
temp_pcap.write(pcap_data)
temp_pcap.close()
choplib.filename = temp_pcap.name
chopui.bind(choplib)
chopui.start()
if chopui.jsonclass == None:
os.unlink(temp_pcap.name)
chopui.join()
choplib.finish()
choplib.join()
return {
'success': False,
'message': 'Lost race condition in chopui. Try again.'
}
# ChopUI must be started before the jsonhandler class is insantiated.
# Tell the class what we are looking for now that it exists.
chopui.jsonclass.parse_options(options)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
os.unlink(temp_pcap.name)
message = ''
# Grab any carved HTTP bodies.
for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
if handle_file(name,
blob,
source,
related_md5=pcap_md5,
user=analyst,
method='ChopShop Filecarver',
md5_digest=md5_digest,
related_type='PCAP'):
# Specifically not using name here as I don't want to deal
# with sanitizing it
message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (
reverse('crits.samples.views.detail', args=[md5_digest
]), md5_digest)
else:
message += "Failed to save file %s." % md5_digest
# Grab any carved SMTP returns.
for blob in chopui.jsonclass.smtp_returns.values():
ret = handle_eml(blob,
source,
None,
analyst,
'ChopShop FileCarver',
related_id=pcap.id,
related_type='PCAP',
relationship_type=RelationshipTypes.RELATED_TO)
if not ret['status']:
message += ret['reason']
continue
message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (
reverse('crits.emails.views.email_detail', args=[
ret['object'].id
]), ret['object'].id, len(ret['attachments'].keys()))
for md5_digest in ret['attachments'].keys():
message += "<a href=\"%s\">%s</a><br />" % (reverse(
'crits.samples.views.detail', args=[md5_digest]), md5_digest)
# Handle raw returns.
for id_, blob in chopui.jsonclass.raw_returns.items():
md5_digest = handle_file(id_,
blob,
source,
related_md5=pcap_md5,
user=analyst,
method='ChopShop Filecarver',
related_type='PCAP')
if md5_digest:
message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (
id_, reverse('crits.samples.views.detail',
args=[md5_digest]), md5_digest)
else:
message += "Failed to save raw %s." % md5_digest
# It's possible to have no files here if nothing matched.
# Still return True as there were no problems.
if not message:
message = 'No files found.'
return {'success': True, 'message': message}
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'Action':
return Action.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'Signature':
return Signature.objects(id=_id).first()
elif type_ == 'SignatureType':
return SignatureType.objects(id=_id).first()
elif type_ == 'SignatureDependency':
return SignatureDependency.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in [
'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator',
'Screenshot'
] and not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(message_id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Signature':
return Signature.objects(md5=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def get_pcap_details(md5, analyst):
"""
Generate the data to render the PCAP details template.
:param md5: The MD5 of the PCAP to get details for.
:type md5: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
pcap = PCAP.objects(md5=md5, source__name__in=sources).first()
if not pcap:
template = "error.html"
args = {"error": "PCAP not yet available or you do not have access to view it."}
else:
pcap.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, pcap.id, "PCAP")
# subscription
subscription = {
"type": "PCAP",
"id": pcap.id,
"subscribed": is_user_subscribed("%s" % analyst, "PCAP", pcap.id),
}
# objects
objects = pcap.sort_objects()
# relationships
relationships = pcap.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {"type": "PCAP", "value": pcap.id}
# comments
comments = {"comments": pcap.get_comments(), "url_key": md5}
# screenshots
screenshots = pcap.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, "PCAP", pcap.id)
# services
manager = crits.service_env.manager
# Assume all PCAPs have the data available
service_list = manager.get_supported_services("PCAP", True)
args = {
"service_list": service_list,
"objects": objects,
"relationships": relationships,
"comments": comments,
"favorite": favorite,
"relationship": relationship,
"subscription": subscription,
"screenshots": screenshots,
"pcap": pcap,
}
return template, args
def pcap_pdml_html(pcap_md5, analyst):
# check to see if there is a File object with the source reference of
# 'tshark_pdml.html'. If there is, return it.
# If not, generate it, save it, and return it.
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return "No PCAP found"
else:
coll = settings.COL_OBJECTS
pdml_obj = None
pdml_html = None
for obj in pcap.obj:
for source in obj.source:
for instance in source.instances:
if instance.reference == "tshark_pdml.html":
pdml_obj = obj
if not pdml_obj:
sc = get_config("MetaCap")
tshark_bin = str(sc["tshark"])
if not os.path.exists(tshark_bin):
pdml_html = "Could not find tshark!"
return {"html": pdml_html}
pcap_data = pcap.filedata.read()
if not pcap_data:
pdml_html = "Could not get PCAP from GridFS: %s" % pcap_md5
return {"html": pdml_html}
# write PCAP to disk
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
pcap_name = temp_pcap.name
temp_pcap.write(pcap_data)
temp_pcap.close()
# use tshark to generate a pdml file
temp_pdml = tempfile.NamedTemporaryFile(delete=False)
args = [tshark_bin, "-n", "-r", pcap_name, "-T", "pdml"]
tshark = Popen(args, stdout=temp_pdml, stderr=PIPE)
tshark_out, tshark_err = tshark.communicate()
if tshark.returncode != 0:
return {"html": "%s, %s" % (tshark_out, tshark_err)}
pdml_name = temp_pdml.name
temp_pdml.seek(0)
# transform PDML into HTML
xsl_file = None
for d in settings.SERVICE_DIRS:
try:
file_dir = "%s/metacap_service" % d
xsl_file = open("%s/pdml2html.xsl" % file_dir, "r")
except IOError:
pass
if not xsl_file:
return {"html": "Could not find XSL."}
parser = etree.XMLParser()
parser.resolvers.add(FileResolver())
save_pdml = False
try:
xml_input = etree.parse(temp_pdml, parser)
xslt_root = etree.parse(xsl_file, parser)
transform = etree.XSLT(xslt_root)
pdml_html = str(transform(xml_input))
save_pdml = True
except Exception:
temp_pdml.close()
# delete PDML file
os.unlink(pdml_name)
os.unlink(pcap_name)
return {"html": "Could not parse/transform PDML output!"}
temp_pdml.close()
# delete PDML file
os.unlink(pdml_name)
os.unlink(pcap_name)
# save pdml_html as an object for this PCAP
if save_pdml:
fn = put_file_gridfs("tshark_pdml.html", pdml_html, collection=coll)
if fn:
m = hashlib.md5()
m.update(pdml_html)
md5 = m.hexdigest()
pcap.add_object(
ObjectTypes.FILE_UPLOAD,
md5,
get_user_organization(analyst),
"MetaCap",
"tshark_pdml.html",
analyst,
)
pcap.save()
else:
# get file from gridfs and return it
obj_md5 = pdml_obj.value
pdml_html = get_file_gridfs(obj_md5, collection=coll)
if not pdml_html:
return {"html": "No file found in GridFS"}
if not pdml_obj:
pcap_objects = pcap.sort_objects()
return {"html": pdml_html, "objects": pcap_objects, "id": pcap.id}
else:
return {"html": pdml_html}
def handle_pcap_file(filename,
data,
source_name,
user=None,
description=None,
related_id=None,
related_md5=None,
related_type=None,
method='',
reference='',
relationship=None,
bucket_list=None,
ticket=None):
"""
Add a PCAP.
:param filename: The filename of the PCAP.
:type filename: str
:param data: The filedata of the PCAP.
:type data: str
:param source_name: The source which provided this PCAP.
:type source_name: str,
:class:`crits.core.crits_mongoengine.EmbeddedSource`,
list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param user: The user adding the PCAP.
:type user: str
:param description: Description of the PCAP.
:type description: str
:param related_id: ObjectId of a top-level object related to this PCAP.
:type related_id: str
:param related_md5: MD5 of a top-level object related to this PCAP.
:type related_md5: str
:param related_type: The CRITs type of the related top-level object.
:type related_type: str
:param method: The method of acquiring this PCAP.
:type method: str
:param reference: A reference to the source of this PCAP.
:type reference: str
:param relationship: The relationship between the parent and the PCAP.
:type relationship: str
:param bucket_list: Bucket(s) to add to this PCAP.
:type bucket_list: str(comma separated) or list.
:param ticket: Ticket(s) to add to this PCAP.
:type ticket: str(comma separated) or list.
:returns: dict with keys:
'success' (boolean),
'message' (str),
'md5' (str) if successful.
"""
if not data:
status = {'success': False, 'message': 'No data object passed in'}
return status
if len(data) <= 0:
status = {'success': False, 'message': 'Data length <= 0'}
return status
if ((related_type and not (related_id or related_md5))
or (not related_type and (related_id or related_md5))):
status = {
'success':
False,
'message':
'Must specify both related_type and related_id or related_md5.'
}
return status
if not source_name:
return {"success": False, "message": "Missing source information."}
related_obj = None
if related_id or related_md5:
if related_id:
related_obj = class_from_id(related_type, related_id)
else:
related_obj = class_from_value(related_type, related_md5)
if not related_obj:
status = {'success': False, 'message': 'Related object not found.'}
return status
# generate md5 and timestamp
md5 = hashlib.md5(data).hexdigest()
timestamp = datetime.datetime.now()
# generate PCAP
is_pcap_new = False
pcap = PCAP.objects(md5=md5).first()
if not pcap:
pcap = PCAP()
pcap.filename = filename
pcap.created = timestamp
pcap.length = len(data)
pcap.description = description
pcap.md5 = md5
is_pcap_new = True
# generate source information and add to pcap
if isinstance(source_name, basestring) and len(source_name) > 0:
s = create_embedded_source(source_name,
method=method,
reference=reference,
analyst=user)
pcap.add_source(s)
elif isinstance(source_name, EmbeddedSource):
pcap.add_source(source_name, method=method, reference=reference)
elif isinstance(source_name, list) and len(source_name) > 0:
for s in source_name:
if isinstance(s, EmbeddedSource):
pcap.add_source(s, method=method, reference=reference)
# add file to GridFS
if not isinstance(pcap.filedata.grid_id, ObjectId):
pcap.add_file_data(data)
if bucket_list:
pcap.add_bucket_list(bucket_list, user)
if ticket:
pcap.add_ticket(ticket, user)
# save pcap
pcap.save(username=user)
# update relationship if a related top-level object is supplied
if related_obj and pcap:
if not relationship:
relationship = "Related_To"
pcap.add_relationship(rel_item=related_obj,
rel_type=relationship,
analyst=user,
get_rels=False)
related_obj.save(username=user)
pcap.save(username=user)
# run pcap triage
if is_pcap_new and data:
pcap.reload()
run_triage(pcap, user)
status = {
'success': True,
'message': 'Uploaded pcap',
'md5': md5,
'id': str(pcap.id),
'object': pcap
}
return status
def get_pcap_details(md5, analyst):
"""
Generate the data to render the PCAP details template.
:param md5: The MD5 of the PCAP to get details for.
:type md5: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
pcap = PCAP.objects(md5=md5, source__name__in=sources).first()
if not pcap:
template = "error.html"
args = {'error': 'PCAP not yet available or you do not have access to view it.'}
else:
pcap.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, pcap.id, 'PCAP')
# subscription
subscription = {
'type': 'PCAP',
'id': pcap.id,
'subscribed': is_user_subscribed("%s" % analyst,
'PCAP', pcap.id),
}
#objects
objects = pcap.sort_objects()
#relationships
relationships = pcap.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {
'type': 'PCAP',
'value': pcap.id
}
#comments
comments = {'comments': pcap.get_comments(),
'url_key': md5}
#screenshots
screenshots = pcap.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'PCAP', pcap.id)
# services
# Assume all PCAPs have the data available
service_list = get_supported_services('PCAP')
# analysis results
service_results = pcap.get_analysis_results()
args = {'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
"subscription": subscription,
"screenshots": screenshots,
"service_results": service_results,
"pcap": pcap}
return template, args
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
'Indicator', 'Screenshot'] and
not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def chopshop_carver(pcap_md5, options, analyst):
# Make sure we can find ChopShop
sc = get_config('ChopShop')
user = get_user_info(analyst)
if not sc:
return {'success': False, 'message': 'Could not find ChopShop service.'}
shop_path = "%s/shop" % str(sc['basedir'])
if not os.path.exists(shop_path):
return {'success': False, 'message': "ChopShop shop path does not exist."}
sys.path.append(shop_path)
import ChopLib as CL
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}
# Until we have an smtp_extractor in ChopShop we have to resort to
# to (ab)using payloads to dump the entire TCP stream and letting
# handle_eml() process everything. We also use the payloads module
# for handling raw carves. If a user wants to do SMTP and raw
# simultaneously it won't work because we can't distinguish one
# payloads module from another.
if options.get('raw', False) and options.get('smtp', False):
return {'success': False, 'message': "Can not process SMTP and raw simultaneously."}
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return {'success': False, 'message': "No PCAP found."}
pcap_data = pcap.filedata.read()
if not pcap_data:
return {'success': False, 'message': "Could not get PCAP from GridFS: %s" % pcap_md5}
source = pcap['source'][0]['name'] # XXX: This kind of sucks...
# Create module string to pass to ChopShop
modules = []
if options.get('http_resp', False) or options.get('http_req', False):
modules.append("http | http_extractor")
if options.get('smtp', False) or options.get('raw', False):
# ChopShop really needs an smtp_extractor, but there's no good
# capability to do that yet. Maybe one day I'll build one. :)
# For now, just use payloads and let handle_eml() sort it out.
#
# Raw carving works exactly the same way, just post-processed
# differently.
modules.append("payloads -b")
if not modules:
return {'success': False, 'message': "No modules specified."}
mod_string = ';'.join(mod for mod in modules)
from ChopLib import ChopLib
from ChopUi import ChopUi
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = str(sc['basedir'])
choplib.modules = mod_string
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# Write the pcap data to a temporary file.
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
temp_pcap.write(pcap_data)
temp_pcap.close()
choplib.filename = temp_pcap.name
chopui.bind(choplib)
chopui.start()
if chopui.jsonclass == None:
os.unlink(temp_pcap.name)
chopui.join()
choplib.finish()
choplib.join()
return {'success': False,
'message': 'Lost race condition in chopui. Try again.'}
# ChopUI must be started before the jsonhandler class is insantiated.
# Tell the class what we are looking for now that it exists.
chopui.jsonclass.parse_options(options)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
os.unlink(temp_pcap.name)
message = ''
# Grab any carved HTTP bodies.
for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
if user.has_access_to(SampleACL.WRITE) and handle_file(name, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', md5_digest=md5_digest, related_type='PCAP'):
# Specifically not using name here as I don't want to deal
# with sanitizing it
message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
else:
message += "Failed to save file %s." % md5_digest
# Grab any carved SMTP returns.
for blob in chopui.jsonclass.smtp_returns.values():
ret = handle_eml(blob, source, None, analyst, 'ChopShop FileCarver',
related_id=pcap.id, related_type='PCAP',
relationship_type=RelationshipTypes.RELATED_TO)
if not ret['status']:
message += ret['reason']
continue
message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (reverse('crits-emails-views-email_detail', args=[ret['object'].id]), ret['object'].id, len(ret['attachments'].keys()))
for md5_digest in ret['attachments'].keys():
message += "<a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
# Handle raw returns.
for id_, blob in chopui.jsonclass.raw_returns.items():
if user.has_access_to(SampleACL.WRITE):
md5_digest = handle_file(id_, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', related_type='PCAP')
else:
md5_digest = None
if md5_digest:
message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (id_, reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
else:
message += "Failed to save raw %s." % md5_digest
# It's possible to have no files here if nothing matched.
# Still return True as there were no problems.
if not message:
message = 'No files found.'
return {'success': True, 'message': message}
def pcap_pdml_html(pcap_md5, analyst):
# check to see if there is a File object with the source reference of
# 'tshark_pdml.html'. If there is, return it.
# If not, generate it, save it, and return it.
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return "No PCAP found"
else:
coll = settings.COL_OBJECTS
pdml_obj = None
pdml_html = None
for obj in pcap.obj:
for source in obj.source:
for instance in source.instances:
if instance.reference == 'tshark_pdml.html':
pdml_obj = obj
if not pdml_obj:
sc = get_config('MetaCap')
tshark_bin = str(sc['tshark'])
if not os.path.exists(tshark_bin):
pdml_html = "Could not find tshark!"
return {'html': pdml_html}
pcap_data = pcap.filedata.read()
if not pcap_data:
pdml_html = "Could not get PCAP from GridFS: %s" % pcap_md5
return {'html': pdml_html}
# write PCAP to disk
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
pcap_name = temp_pcap.name
temp_pcap.write(pcap_data)
temp_pcap.close()
# use tshark to generate a pdml file
temp_pdml = tempfile.NamedTemporaryFile(delete=False)
args = [tshark_bin, "-n", "-r", pcap_name, "-T", "pdml"]
tshark = Popen(args, stdout=temp_pdml, stderr=PIPE)
tshark_out, tshark_err = tshark.communicate()
if tshark.returncode != 0:
return {'html': "%s, %s" % (tshark_out,tshark_err)}
pdml_name = temp_pdml.name
temp_pdml.seek(0)
# transform PDML into HTML
xsl_file = None
for d in settings.SERVICE_DIRS:
try:
file_dir = "%s/metacap_service" % d
xsl_file = open('%s/pdml2html.xsl' % file_dir, 'r')
except IOError:
pass
if not xsl_file:
return {'html': 'Could not find XSL.'}
parser = etree.XMLParser()
parser.resolvers.add(FileResolver())
save_pdml = False
try:
xml_input = etree.parse(temp_pdml, parser)
xslt_root = etree.parse(xsl_file, parser)
transform = etree.XSLT(xslt_root)
pdml_html = str(transform(xml_input))
save_pdml = True
except Exception:
return {'html': 'Could not parse/transform PDML output!'}
temp_pdml.close()
# delete PDML file
os.unlink(pdml_name)
os.unlink(pcap_name)
# save pdml_html as an object for this PCAP
if save_pdml:
fn = put_file_gridfs('tshark_pdml.html', pdml_html, collection=coll)
if fn:
m = hashlib.md5()
m.update(pdml_html)
md5 = m.hexdigest()
pcap.add_object("Artifact",
"File",
md5,
get_user_organization(analyst),
"MetaCap",
'tshark_pdml.html',
analyst)
pcap.save()
else:
# get file from gridfs and return it
obj_md5 = pdml_obj.value
pdml_html = get_file_gridfs(obj_md5, collection=coll)
if not pdml_html:
return {'html': 'No file found in GridFS'}
if not pdml_obj:
pcap_objects = pcap.sort_objects()
return {'html': pdml_html, 'objects': pcap_objects, 'id': pcap.id}
else:
return {'html': pdml_html}
def get_pcap_details(md5, analyst):
"""
Generate the data to render the PCAP details template.
:param md5: The MD5 of the PCAP to get details for.
:type md5: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
sources = user_sources(analyst)
pcap = PCAP.objects(md5=md5, source__name__in=sources).first()
if not pcap:
template = "error.html"
args = {
'error':
'PCAP not yet available or you do not have access to view it.'
}
else:
pcap.sanitize("%s" % analyst)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, pcap.id, 'PCAP')
# subscription
subscription = {
'type': 'PCAP',
'id': pcap.id,
'subscribed': is_user_subscribed("%s" % analyst, 'PCAP', pcap.id),
}
#objects
objects = pcap.sort_objects()
#relationships
relationships = pcap.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'PCAP', 'value': pcap.id}
#comments
comments = {'comments': pcap.get_comments(), 'url_key': md5}
#screenshots
screenshots = pcap.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'PCAP', pcap.id)
# services
# Assume all PCAPs have the data available
service_list = get_supported_services('PCAP')
# analysis results
service_results = pcap.get_analysis_results()
args = {
'service_list': service_list,
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
"subscription": subscription,
"screenshots": screenshots,
"service_results": service_results,
"pcap": pcap
}
return template, args
def pcap_tcpdump(pcap_md5, form, analyst):
flag_list = []
cleaned_data = form.cleaned_data
# Make sure we can find tcpdump
sc = get_config('MetaCap')
tcpdump_bin = str(sc['tcpdump'])
if not os.path.exists(tcpdump_bin):
tcpdump_output = "Could not find tcpdump!"
return tcpdump_output
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return "No PCAP found"
pcap_data = pcap.filedata.read()
if not pcap_data:
return "Could not get PCAP from GridFS: %s" % pcap_md5
# Use the filename if it's there, otherwise the md5.
# This is used for the description of the carved sample.
if pcap.filename:
pcap_filename = pcap.filename
else:
pcap_filename = pcap_md5
# Setup tcpdump arguments
if cleaned_data['sequence']:
flag_list.append("-S")
if cleaned_data['timestamp']:
flag_list.append("%s" % cleaned_data['timestamp'])
if cleaned_data['verbose']:
flag_list.append("%s" % cleaned_data['verbose'])
if cleaned_data['data']:
flag_list.append("%s" % cleaned_data['data'])
# force -nN
flag_list.append("-nN")
# if we need to carve
if cleaned_data['carve']:
if not cleaned_data['bpf']:
return "Must supply a BPF filter to carve."
new_pcap = tempfile.NamedTemporaryFile(delete=False)
flag_list.append("-w")
flag_list.append(new_pcap.name)
if cleaned_data['bpf']:
flag_list.append('%s' % str(cleaned_data['bpf'].replace('"', '')))
# write PCAP to disk
# temp_out collects stdout and stderr
# temp_pcap is the pcap to read
# new_pcap is the pcap being written if carving
temp_out = tempfile.NamedTemporaryFile(delete=False)
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
pcap_name = temp_pcap.name
temp_pcap.write(pcap_data)
temp_pcap.close()
args = [tcpdump_bin, '-r', temp_pcap.name] + flag_list
tcpdump = Popen(args, stdout=temp_out, stderr=STDOUT)
tcpdump.communicate()
out_name = temp_out.name
temp_out.seek(0)
tcpdump_output = ''
for line in iter(temp_out):
tcpdump_output += "%s" % line
temp_out.close()
#delete temp files
os.unlink(pcap_name)
os.unlink(out_name)
if cleaned_data['carve']:
new_pcap_data = new_pcap.read()
if len(new_pcap_data) > 24: # pcap-ng will change this.
m = hashlib.md5()
m.update(new_pcap_data)
md5 = m.hexdigest()
org = get_user_organization(analyst)
result = handle_pcap_file("%s.pcap" % md5,
new_pcap_data,
org,
user=analyst,
description="%s of %s" % (cleaned_data['bpf'], pcap_filename),
parent_id=pcap.id,
parent_type="PCAP",
method="MetaCap Tcpdumper")
if result['success']:
tcpdump_output = "<a href=\"%s\">View new pcap.</a>" % reverse('crits.pcaps.views.pcap_details', args=[result['md5']])
else:
tcpdump_output = result['message']
else:
tcpdump_output = "No packets matched the filter."
os.unlink(new_pcap.name)
return tcpdump_output
def pcap_tcpdump(pcap_md5, form, analyst):
flag_list = []
cleaned_data = form.cleaned_data
# Make sure we can find tcpdump
sc = get_config("MetaCap")
tcpdump_bin = str(sc["tcpdump"])
if not os.path.exists(tcpdump_bin):
tcpdump_output = "Could not find tcpdump!"
return tcpdump_output
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return "No PCAP found"
pcap_data = pcap.filedata.read()
if not pcap_data:
return "Could not get PCAP from GridFS: %s" % pcap_md5
# Use the filename if it's there, otherwise the md5.
# This is used for the description of the carved sample.
if pcap.filename:
pcap_filename = pcap.filename
else:
pcap_filename = pcap_md5
# Setup tcpdump arguments
if cleaned_data["sequence"]:
flag_list.append("-S")
if cleaned_data["timestamp"]:
flag_list.append("%s" % cleaned_data["timestamp"])
if cleaned_data["verbose"]:
flag_list.append("%s" % cleaned_data["verbose"])
if cleaned_data["data"]:
flag_list.append("%s" % cleaned_data["data"])
# force -nN
flag_list.append("-nN")
# if we need to carve
if cleaned_data["carve"]:
if not cleaned_data["bpf"]:
return "Must supply a BPF filter to carve."
new_pcap = tempfile.NamedTemporaryFile(delete=False)
flag_list.append("-w")
flag_list.append(new_pcap.name)
if cleaned_data["bpf"]:
flag_list.append("%s" % str(cleaned_data["bpf"].replace('"', "")))
# write PCAP to disk
# temp_out collects stdout and stderr
# temp_pcap is the pcap to read
# new_pcap is the pcap being written if carving
temp_out = tempfile.NamedTemporaryFile(delete=False)
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
pcap_name = temp_pcap.name
temp_pcap.write(pcap_data)
temp_pcap.close()
args = [tcpdump_bin, "-r", temp_pcap.name] + flag_list
tcpdump = Popen(args, stdout=temp_out, stderr=STDOUT)
tcpdump.communicate()
out_name = temp_out.name
temp_out.seek(0)
tcpdump_output = ""
for line in iter(temp_out):
tcpdump_output += "%s" % line
temp_out.close()
# delete temp files
os.unlink(pcap_name)
os.unlink(out_name)
if cleaned_data["carve"]:
new_pcap_data = new_pcap.read()
if len(new_pcap_data) > 24: # pcap-ng will change this.
m = hashlib.md5()
m.update(new_pcap_data)
md5 = m.hexdigest()
org = get_user_organization(analyst)
result = handle_pcap_file(
"%s.pcap" % md5,
new_pcap_data,
org,
user=analyst,
description="%s of %s" % (cleaned_data["bpf"], pcap_filename),
parent_id=pcap.id,
parent_type="PCAP",
method="MetaCap Tcpdumper",
)
if result["success"]:
tcpdump_output = '<a href="%s">View new pcap.</a>' % reverse(
"crits.pcaps.views.pcap_details", args=[result["md5"]]
)
else:
tcpdump_output = result["message"]
else:
tcpdump_output = "No packets matched the filter."
os.unlink(new_pcap.name)
return tcpdump_output
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# Quick fail
if not _id or not type_:
return None
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode("utf8")):
return None
if type_ == "Actor":
return Actor.objects(id=_id).first()
elif type_ == "Backdoor":
return Backdoor.objects(id=_id).first()
elif type_ == "ActorThreatIdentifier":
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == "Campaign":
return Campaign.objects(id=_id).first()
elif type_ == "Certificate":
return Certificate.objects(id=_id).first()
elif type_ == "Comment":
return Comment.objects(id=_id).first()
elif type_ == "Domain":
return Domain.objects(id=_id).first()
elif type_ == "Email":
return Email.objects(id=_id).first()
elif type_ == "Event":
return Event.objects(id=_id).first()
elif type_ == "Exploit":
return Exploit.objects(id=_id).first()
elif type_ == "Indicator":
return Indicator.objects(id=_id).first()
elif type_ == "Action":
return Action.objects(id=_id).first()
elif type_ == "IP":
return IP.objects(id=_id).first()
elif type_ == "PCAP":
return PCAP.objects(id=_id).first()
elif type_ == "RawData":
return RawData.objects(id=_id).first()
elif type_ == "RawDataType":
return RawDataType.objects(id=_id).first()
elif type_ == "Sample":
return Sample.objects(id=_id).first()
elif type_ == "Signature":
return Signature.objects(id=_id).first()
elif type_ == "SignatureType":
return SignatureType.objects(id=_id).first()
elif type_ == "SignatureDependency":
return SignatureDependency.objects(id=_id).first()
elif type_ == "SourceAccess":
return SourceAccess.objects(id=_id).first()
elif type_ == "Screenshot":
return Screenshot.objects(id=_id).first()
elif type_ == "Target":
return Target.objects(id=_id).first()
elif type_ == "UserRole":
return UserRole.objects(id=_id).first()
else:
return None