Esempio n. 1
0
def class_from_value(type_, value):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param value: The value to search for.
    :type value: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.indicators.indicator import Indicator
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.targets.target import Target

    if type_ == 'Campaign':
        return Campaign.objects(name=value).first()
    elif type_ == 'Certificate':
        return Certificate.objects(md5=value).first()
    elif type_ == 'Comment':
        return Comment.objects(id=value).first()
    elif type_ == 'Domain':
        return Domain.objects(domain=value).first()
    elif type_ == 'Email':
        return Email.objects(id=value).first()
    elif type_ == 'Event':
        return Event.objects(id=value).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=value).first()
    elif type_ == 'IP':
        return IP.objects(ip=value).first()
    elif type_ == 'PCAP':
        return PCAP.objects(md5=value).first()
    elif type_ == 'RawData':
        return RawData.objects(md5=value).first()
    elif type_ == 'Sample':
        return Sample.objects(md5=value).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=value).first()
    elif type_ == 'Target':
        return Target.objects(email_address=value).first()
    else:
        return None
Esempio n. 2
0
def class_from_value(type_, value):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param value: The value to search for.
    :type value: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.indicators.indicator import Indicator
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.targets.target import Target

    if type_ == 'Campaign':
        return Campaign.objects(name=value).first()
    elif type_ == 'Certificate':
        return Certificate.objects(md5=value).first()
    elif type_ == 'Comment':
        return Comment.objects(id=value).first()
    elif type_ == 'Domain':
        return Domain.objects(domain=value).first()
    elif type_ == 'Email':
        return Email.objects(id=value).first()
    elif type_ == 'Event':
        return Event.objects(id=value).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=value).first()
    elif type_ == 'IP':
        return IP.objects(ip=value).first()
    elif type_ == 'PCAP':
        return PCAP.objects(md5=value).first()
    elif type_ == 'RawData':
        return RawData.objects(md5=value).first()
    elif type_ == 'Sample':
        return Sample.objects(md5=value).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=value).first()
    elif type_ == 'Target':
        return Target.objects(email_address=value).first()
    else:
        return None
Esempio n. 3
0
    def _delete_all_analysis_results(self, md5_digest, service_name):
        """
        Delete all analysis results for this service.
        """

        obj = Sample.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [
                a for a in obj.analysis if a.service_name != service_name
            ]
            obj.save()
        obj = PCAP.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [
                a for a in obj.analysis if a.service_name != service_name
            ]
            obj.save()
        obj = Certificate.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [
                a for a in obj.analysis if a.service_name != service_name
            ]
            obj.save()
        obj = RawData.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [
                a for a in obj.analysis if a.service_name != service_name
            ]
            obj.save()
        obj = Event.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [
                a for a in obj.analysis if a.service_name != service_name
            ]
            obj.save()
        obj = Indicator.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [
                a for a in obj.analysis if a.service_name != service_name
            ]
            obj.save()
        obj = Domain.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [
                a for a in obj.analysis if a.service_name != service_name
            ]
            obj.save()
        obj = IP.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [
                a for a in obj.analysis if a.service_name != service_name
            ]
            obj.save()
Esempio n. 4
0
def delete_pcap(pcap_md5, username=None):
    """
    Delete a PCAP.

    :param pcap_md5: The MD5 of the PCAP to delete.
    :type pcap_md5: str
    :param username: The user deleting the pcap.
    :type username: str
    :returns: True, False
    """

    pcap = PCAP.objects(md5=pcap_md5).first()
    if pcap:
        pcap.delete(username=username)
        return True
    else:
        return False
Esempio n. 5
0
def delete_pcap(pcap_md5, username=None):
    """
    Delete a PCAP.

    :param pcap_md5: The MD5 of the PCAP to delete.
    :type pcap_md5: str
    :param username: The user deleting the pcap.
    :type username: str
    :returns: True, False
    """

    pcap = PCAP.objects(md5=pcap_md5).first()
    if pcap:
        pcap.delete(username=username)
        return True
    else:
        return False
Esempio n. 6
0
    def create_pcap_context(self, identifier, username):
        # .only() is currently broken in MongoEngine :(
        #fields = ('filename', 'length', 'filedata')
        #pcap = PCAP.objects(id=identifier).only(*fields).first()
        pcap = PCAP.objects(id=identifier).first()

        if not pcap:
            raise ValueError("PCAP not found in database")

        data = pcap.filedata.read()
        if not data:
            raise ValueError("PCAP not found in GridFS")

        pcap_md5 = pcap.md5
        self._check_length(data, getattr(pcap, 'length', 0))

        return PCAPContext(username, data, pcap_md5, pcap.to_dict())
Esempio n. 7
0
    def create_pcap_context(self, identifier, username):
        # .only() is currently broken in MongoEngine :(
        #fields = ('filename', 'length', 'filedata')
        #pcap = PCAP.objects(id=identifier).only(*fields).first()
        pcap = PCAP.objects(id=identifier).first()

        if not pcap:
            raise ValueError("PCAP not found in database")

        data = pcap.filedata.read()
        if not data:
            raise ValueError("PCAP not found in GridFS")

        pcap_md5 = pcap.md5
        self._check_length(data, getattr(pcap, 'length', 0))

        return PCAPContext(username, data, pcap_md5, pcap.to_dict())
Esempio n. 8
0
def update_pcap_description(md5, description, analyst):
    """
    Update a PCAP description.

    :param md5: The MD5 of the PCAP to update.
    :type md5: str
    :param description: The new description.
    :type description: str
    :param analyst: The user updating the description.
    :type analyst: str
    :returns: None, ValidationError
    """

    pcap = PCAP.objects(md5=md5).first()
    pcap.description = description
    try:
        pcap.save(username=analyst)
        return None
    except ValidationError, e:
        return e
Esempio n. 9
0
def update_pcap_description(md5, description, analyst):
    """
    Update a PCAP description.

    :param md5: The MD5 of the PCAP to update.
    :type md5: str
    :param description: The new description.
    :type description: str
    :param analyst: The user updating the description.
    :type analyst: str
    :returns: None, ValidationError
    """

    pcap = PCAP.objects(md5=md5).first()
    pcap.description = description
    try:
        pcap.save(username=analyst)
        return None
    except ValidationError, e:
        return e
Esempio n. 10
0
    def _delete_all_analysis_results(self, md5_digest, service_name):
        """
        Delete all analysis results for this service.
        """

        obj = Sample.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = PCAP.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = Certificate.objects(md5=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = RawData.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = Event.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = Indicator.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = Domain.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
        obj = IP.objects(id=md5_digest).first()
        if obj:
            obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
            obj.save()
Esempio n. 11
0
def handle_pcap_file(filename, data, source_name, user=None,
                     description=None, related_id=None, related_md5=None,
                     related_type=None, method=None, reference=None,
                     relationship=None, bucket_list=None, ticket=None):
    """
    Add a PCAP.

    :param filename: The filename of the PCAP.
    :type filename: str
    :param data: The filedata of the PCAP.
    :type data: str
    :param source_name: The source which provided this PCAP.
    :type source_name: str,
                       :class:`crits.core.crits_mongoengine.EmbeddedSource`,
                       list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param user: The user adding the PCAP.
    :type user: str
    :param description: Description of the PCAP.
    :type description: str
    :param related_id: ObjectId of a top-level object related to this PCAP.
    :type related_id: str
    :param related_md5: MD5 of a top-level object related to this PCAP.
    :type related_md5: str
    :param related_type: The CRITs type of the related top-level object.
    :type related_type: str
    :param method: The method of acquiring this PCAP.
    :type method: str
    :param reference: A reference to the source of this PCAP.
    :type reference: str
    :param relationship: The relationship between the parent and the PCAP.
    :type relationship: str
    :param bucket_list: Bucket(s) to add to this PCAP.
    :type bucket_list: str(comma separated) or list.
    :param ticket: Ticket(s) to add to this PCAP.
    :type ticket: str(comma separated) or list.
    :returns: dict with keys:
              'success' (boolean),
              'message' (str),
              'md5' (str) if successful.
    """

    if not data:
        status = {
            'success':   False,
            'message':  'No data object passed in'
        }
        return status
    if len(data) <= 0:
        status = {
            'success':   False,
            'message':  'Data length <= 0'
        }
        return status
    if ((related_type and not (related_id or related_md5)) or
        (not related_type and (related_id or related_md5))):
        status = {
            'success':   False,
            'message':  'Must specify both related_type and related_id or related_md5.'
        }
        return status

    if not source_name:
        return {"success" : False, "message" : "Missing source information."}

    related_obj = None
    if related_id or related_md5:
        if related_id:
            related_obj = class_from_id(related_type, related_id)
        else:
            related_obj = class_from_value(related_type, related_md5)
        if not related_obj:
            status = {
                'success': False,
                'message': 'Related object not found.'
            }
            return status


    # generate md5 and timestamp
    md5 = hashlib.md5(data).hexdigest()
    timestamp = datetime.datetime.now()

    # generate PCAP
    is_pcap_new = False
    pcap = PCAP.objects(md5=md5).first()
    if not pcap:
        pcap = PCAP()
        pcap.filename = filename
        pcap.created = timestamp
        pcap.length = len(data)
        pcap.description = description
        pcap.md5 = md5
        is_pcap_new = True

    # generate source information and add to pcap
    if isinstance(source_name, basestring) and len(source_name) > 0:
        s = create_embedded_source(source_name,
                                   method=method,
                                   reference=reference,
                                   analyst=user)
        pcap.add_source(s)
    elif isinstance(source_name, EmbeddedSource):
        pcap.add_source(source_name, method=method, reference=reference)
    elif isinstance(source_name, list) and len(source_name) > 0:
        for s in source_name:
            if isinstance(s, EmbeddedSource):
                pcap.add_source(s, method=method, reference=reference)

    # add file to GridFS
    if not isinstance(pcap.filedata.grid_id, ObjectId):
        pcap.add_file_data(data)

    if bucket_list:
        pcap.add_bucket_list(bucket_list, user)

    if ticket:
        pcap.add_ticket(ticket, user)

    # save pcap
    pcap.save(username=user)

    # update relationship if a related top-level object is supplied
    if related_obj and pcap:
        if not relationship:
            relationship = "Related_To"
        pcap.add_relationship(rel_item=related_obj,
                              rel_type=relationship,
                              analyst=user,
                              get_rels=False)
        related_obj.save(username=user)
        pcap.save(username=user)

    # run pcap triage
    if is_pcap_new and data:
        pcap.reload()
        run_triage(pcap, user)

    status = {
        'success':      True,
        'message':      'Uploaded pcap',
        'md5':          md5,
        'id':           str(pcap.id),
    }

    return status
Esempio n. 12
0
def chopshop_carver(pcap_md5, options, analyst):
    # Make sure we can find ChopShop
    sc = get_config('ChopShop')
    if not sc:
        return {
            'success': False,
            'message': 'Could not find ChopShop service.'
        }

    shop_path = "%s/shop" % str(sc['basedir'])
    if not os.path.exists(shop_path):
        return {
            'success': False,
            'message': "ChopShop shop path does not exist."
        }

    sys.path.append(shop_path)
    import ChopLib as CL
    if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
        return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}

    # Until we have an smtp_extractor in ChopShop we have to resort to
    # to (ab)using payloads to dump the entire TCP stream and letting
    # handle_eml() process everything. We also use the payloads module
    # for handling raw carves. If a user wants to do SMTP and raw
    # simultaneously it won't work because we can't distinguish one
    # payloads module from another.
    if options.get('raw', False) and options.get('smtp', False):
        return {
            'success': False,
            'message': "Can not process SMTP and raw simultaneously."
        }

    # Make sure we have a PCAP to work with
    pcap = PCAP.objects(md5=pcap_md5).first()
    if not pcap:
        return {'success': False, 'message': "No PCAP found."}
    pcap_data = pcap.filedata.read()
    if not pcap_data:
        return {
            'success': False,
            'message': "Could not get PCAP from GridFS: %s" % pcap_md5
        }

    source = pcap['source'][0]['name']  # XXX: This kind of sucks...

    # Create module string to pass to ChopShop
    modules = []
    if options.get('http_resp', False) or options.get('http_req', False):
        modules.append("http | http_extractor")

    if options.get('smtp', False) or options.get('raw', False):
        # ChopShop really needs an smtp_extractor, but there's no good
        # capability to do that yet. Maybe one day I'll build one. :)
        # For now, just use payloads and let handle_eml() sort it out.
        #
        # Raw carving works exactly the same way, just post-processed
        # differently.
        modules.append("payloads -b")

    if not modules:
        return {'success': False, 'message': "No modules specified."}

    mod_string = ';'.join(mod for mod in modules)

    from ChopLib import ChopLib
    from ChopUi import ChopUi

    choplib = ChopLib()
    chopui = ChopUi()

    choplib.base_dir = str(sc['basedir'])

    choplib.modules = mod_string

    chopui.jsonout = jsonhandler
    choplib.jsonout = True

    # ChopShop (because of pynids) needs to read a file off disk.
    # Write the pcap data to a temporary file.
    temp_pcap = tempfile.NamedTemporaryFile(delete=False)
    temp_pcap.write(pcap_data)
    temp_pcap.close()

    choplib.filename = temp_pcap.name
    chopui.bind(choplib)
    chopui.start()

    if chopui.jsonclass == None:
        os.unlink(temp_pcap.name)
        chopui.join()
        choplib.finish()
        choplib.join()
        return {
            'success': False,
            'message': 'Lost race condition in chopui. Try again.'
        }

    # ChopUI must be started before the jsonhandler class is insantiated.
    # Tell the class what we are looking for now that it exists.
    chopui.jsonclass.parse_options(options)

    choplib.start()

    while chopui.is_alive():
        time.sleep(.1)

    chopui.join()
    choplib.finish()
    choplib.join()

    os.unlink(temp_pcap.name)

    message = ''

    # Grab any carved HTTP bodies.
    for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
        if handle_file(name,
                       blob,
                       source,
                       related_md5=pcap_md5,
                       user=analyst,
                       method='ChopShop Filecarver',
                       md5_digest=md5_digest,
                       related_type='PCAP'):
            # Specifically not using name here as I don't want to deal
            # with sanitizing it
            message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (
                reverse('crits.samples.views.detail', args=[md5_digest
                                                            ]), md5_digest)
        else:
            message += "Failed to save file %s." % md5_digest

    # Grab any carved SMTP returns.
    for blob in chopui.jsonclass.smtp_returns.values():
        ret = handle_eml(blob,
                         source,
                         None,
                         analyst,
                         'ChopShop FileCarver',
                         related_id=pcap.id,
                         related_type='PCAP',
                         relationship_type=RelationshipTypes.RELATED_TO)
        if not ret['status']:
            message += ret['reason']
            continue

        message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (
            reverse('crits.emails.views.email_detail', args=[
                ret['object'].id
            ]), ret['object'].id, len(ret['attachments'].keys()))

        for md5_digest in ret['attachments'].keys():
            message += "<a href=\"%s\">%s</a><br />" % (reverse(
                'crits.samples.views.detail', args=[md5_digest]), md5_digest)

    # Handle raw returns.
    for id_, blob in chopui.jsonclass.raw_returns.items():
        md5_digest = handle_file(id_,
                                 blob,
                                 source,
                                 related_md5=pcap_md5,
                                 user=analyst,
                                 method='ChopShop Filecarver',
                                 related_type='PCAP')
        if md5_digest:
            message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (
                id_, reverse('crits.samples.views.detail',
                             args=[md5_digest]), md5_digest)
        else:
            message += "Failed to save raw %s." % md5_digest

    # It's possible to have no files here if nothing matched.
    # Still return True as there were no problems.
    if not message:
        message = 'No files found.'
    return {'success': True, 'message': message}
Esempio n. 13
0
def class_from_id(type_, _id):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param _id: The ObjectId to search for.
    :type _id: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.actors.actor import ActorThreatIdentifier, Actor
    from crits.backdoors.backdoor import Backdoor
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.core.crits_mongoengine import Action
    from crits.core.source_access import SourceAccess
    from crits.core.user_role import UserRole
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.exploits.exploit import Exploit
    from crits.indicators.indicator import Indicator
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData, RawDataType
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.signatures.signature import Signature, SignatureType, SignatureDependency
    from crits.targets.target import Target

    if not _id:
        return None

    # make sure it's a string
    _id = str(_id)

    # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
    # the queries below will raise a ValidationError exception.
    if not ObjectId.is_valid(_id.decode('utf8')):
        return None

    if type_ == 'Actor':
        return Actor.objects(id=_id).first()
    elif type_ == 'Backdoor':
        return Backdoor.objects(id=_id).first()
    elif type_ == 'ActorThreatIdentifier':
        return ActorThreatIdentifier.objects(id=_id).first()
    elif type_ == 'Campaign':
        return Campaign.objects(id=_id).first()
    elif type_ == 'Certificate':
        return Certificate.objects(id=_id).first()
    elif type_ == 'Comment':
        return Comment.objects(id=_id).first()
    elif type_ == 'Domain':
        return Domain.objects(id=_id).first()
    elif type_ == 'Email':
        return Email.objects(id=_id).first()
    elif type_ == 'Event':
        return Event.objects(id=_id).first()
    elif type_ == 'Exploit':
        return Exploit.objects(id=_id).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=_id).first()
    elif type_ == 'Action':
        return Action.objects(id=_id).first()
    elif type_ == 'IP':
        return IP.objects(id=_id).first()
    elif type_ == 'PCAP':
        return PCAP.objects(id=_id).first()
    elif type_ == 'RawData':
        return RawData.objects(id=_id).first()
    elif type_ == 'RawDataType':
        return RawDataType.objects(id=_id).first()
    elif type_ == 'Sample':
        return Sample.objects(id=_id).first()
    elif type_ == 'Signature':
        return Signature.objects(id=_id).first()
    elif type_ == 'SignatureType':
        return SignatureType.objects(id=_id).first()
    elif type_ == 'SignatureDependency':
        return SignatureDependency.objects(id=_id).first()
    elif type_ == 'SourceAccess':
        return SourceAccess.objects(id=_id).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=_id).first()
    elif type_ == 'Target':
        return Target.objects(id=_id).first()
    elif type_ == 'UserRole':
        return UserRole.objects(id=_id).first()
    else:
        return None
Esempio n. 14
0
def class_from_value(type_, value):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param value: The value to search for.
    :type value: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.actors.actor import ActorThreatIdentifier, Actor
    from crits.backdoors.backdoor import Backdoor
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.exploits.exploit import Exploit
    from crits.indicators.indicator import Indicator
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.signatures.signature import Signature
    from crits.targets.target import Target

    # Make sure value is a string...
    value = str(value)

    # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
    # the queries below will raise a ValidationError exception.
    if (type_ in [
            'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator',
            'Screenshot'
    ] and not ObjectId.is_valid(value.decode('utf8'))):
        return None

    if type_ == 'Actor':
        return Actor.objects(name=value).first()
    if type_ == 'Backdoor':
        return Backdoor.objects(id=value).first()
    elif type_ == 'ActorThreatIdentifier':
        return ActorThreatIdentifier.objects(name=value).first()
    elif type_ == 'Campaign':
        return Campaign.objects(name=value).first()
    elif type_ == 'Certificate':
        return Certificate.objects(md5=value).first()
    elif type_ == 'Comment':
        return Comment.objects(id=value).first()
    elif type_ == 'Domain':
        return Domain.objects(domain=value).first()
    elif type_ == 'Email':
        return Email.objects(message_id=value).first()
    elif type_ == 'Event':
        return Event.objects(id=value).first()
    elif type_ == 'Exploit':
        return Exploit.objects(id=value).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=value).first()
    elif type_ == 'IP':
        return IP.objects(ip=value).first()
    elif type_ == 'PCAP':
        return PCAP.objects(md5=value).first()
    elif type_ == 'RawData':
        return RawData.objects(md5=value).first()
    elif type_ == 'Sample':
        return Sample.objects(md5=value).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=value).first()
    elif type_ == 'Signature':
        return Signature.objects(md5=value).first()
    elif type_ == 'Target':
        target = Target.objects(email_address=value).first()
        if target:
            return target
        else:
            return Target.objects(email_address__iexact=value).first()
    else:
        return None
Esempio n. 15
0
def get_pcap_details(md5, analyst):
    """
    Generate the data to render the PCAP details template.

    :param md5: The MD5 of the PCAP to get details for.
    :type md5: str
    :param analyst: The user requesting this information.
    :type analyst: str
    :returns: template (str), arguments (dict)
    """

    template = None
    sources = user_sources(analyst)
    pcap = PCAP.objects(md5=md5, source__name__in=sources).first()
    if not pcap:
        template = "error.html"
        args = {"error": "PCAP not yet available or you do not have access to view it."}
    else:

        pcap.sanitize("%s" % analyst)

        # remove pending notifications for user
        remove_user_from_notification("%s" % analyst, pcap.id, "PCAP")

        # subscription
        subscription = {
            "type": "PCAP",
            "id": pcap.id,
            "subscribed": is_user_subscribed("%s" % analyst, "PCAP", pcap.id),
        }

        # objects
        objects = pcap.sort_objects()

        # relationships
        relationships = pcap.sort_relationships("%s" % analyst, meta=True)

        # relationship
        relationship = {"type": "PCAP", "value": pcap.id}

        # comments
        comments = {"comments": pcap.get_comments(), "url_key": md5}

        # screenshots
        screenshots = pcap.get_screenshots(analyst)

        # favorites
        favorite = is_user_favorite("%s" % analyst, "PCAP", pcap.id)

        # services
        manager = crits.service_env.manager
        # Assume all PCAPs have the data available
        service_list = manager.get_supported_services("PCAP", True)

        args = {
            "service_list": service_list,
            "objects": objects,
            "relationships": relationships,
            "comments": comments,
            "favorite": favorite,
            "relationship": relationship,
            "subscription": subscription,
            "screenshots": screenshots,
            "pcap": pcap,
        }

    return template, args
Esempio n. 16
0
def pcap_pdml_html(pcap_md5, analyst):
    # check to see if there is a File object with the source reference of
    # 'tshark_pdml.html'. If there is, return it.
    # If not, generate it, save it, and return it.
    pcap = PCAP.objects(md5=pcap_md5).first()
    if not pcap:
        return "No PCAP found"
    else:
        coll = settings.COL_OBJECTS
        pdml_obj = None
        pdml_html = None
        for obj in pcap.obj:
            for source in obj.source:
                for instance in source.instances:
                    if instance.reference == "tshark_pdml.html":
                        pdml_obj = obj
        if not pdml_obj:
            sc = get_config("MetaCap")
            tshark_bin = str(sc["tshark"])
            if not os.path.exists(tshark_bin):
                pdml_html = "Could not find tshark!"
                return {"html": pdml_html}

            pcap_data = pcap.filedata.read()
            if not pcap_data:
                pdml_html = "Could not get PCAP from GridFS: %s" % pcap_md5
                return {"html": pdml_html}

            # write PCAP to disk
            temp_pcap = tempfile.NamedTemporaryFile(delete=False)
            pcap_name = temp_pcap.name
            temp_pcap.write(pcap_data)
            temp_pcap.close()

            # use tshark to generate a pdml file
            temp_pdml = tempfile.NamedTemporaryFile(delete=False)
            args = [tshark_bin, "-n", "-r", pcap_name, "-T", "pdml"]
            tshark = Popen(args, stdout=temp_pdml, stderr=PIPE)
            tshark_out, tshark_err = tshark.communicate()
            if tshark.returncode != 0:
                return {"html": "%s, %s" % (tshark_out, tshark_err)}
            pdml_name = temp_pdml.name
            temp_pdml.seek(0)

            # transform PDML into HTML
            xsl_file = None
            for d in settings.SERVICE_DIRS:
                try:
                    file_dir = "%s/metacap_service" % d
                    xsl_file = open("%s/pdml2html.xsl" % file_dir, "r")
                except IOError:
                    pass
            if not xsl_file:
                return {"html": "Could not find XSL."}

            parser = etree.XMLParser()
            parser.resolvers.add(FileResolver())
            save_pdml = False
            try:
                xml_input = etree.parse(temp_pdml, parser)
                xslt_root = etree.parse(xsl_file, parser)
                transform = etree.XSLT(xslt_root)
                pdml_html = str(transform(xml_input))
                save_pdml = True
            except Exception:
                temp_pdml.close()
                # delete PDML file
                os.unlink(pdml_name)
                os.unlink(pcap_name)
                return {"html": "Could not parse/transform PDML output!"}

            temp_pdml.close()

            # delete PDML file
            os.unlink(pdml_name)
            os.unlink(pcap_name)

            #  save pdml_html as an object for this PCAP
            if save_pdml:
                fn = put_file_gridfs("tshark_pdml.html", pdml_html, collection=coll)
                if fn:
                    m = hashlib.md5()
                    m.update(pdml_html)
                    md5 = m.hexdigest()
                    pcap.add_object(
                        ObjectTypes.FILE_UPLOAD,
                        md5,
                        get_user_organization(analyst),
                        "MetaCap",
                        "tshark_pdml.html",
                        analyst,
                    )
                    pcap.save()
        else:
            # get file from gridfs and return it
            obj_md5 = pdml_obj.value
            pdml_html = get_file_gridfs(obj_md5, collection=coll)
            if not pdml_html:
                return {"html": "No file found in GridFS"}
        if not pdml_obj:
            pcap_objects = pcap.sort_objects()
            return {"html": pdml_html, "objects": pcap_objects, "id": pcap.id}
        else:
            return {"html": pdml_html}
Esempio n. 17
0
def handle_pcap_file(filename,
                     data,
                     source_name,
                     user=None,
                     description=None,
                     related_id=None,
                     related_md5=None,
                     related_type=None,
                     method='',
                     reference='',
                     relationship=None,
                     bucket_list=None,
                     ticket=None):
    """
    Add a PCAP.

    :param filename: The filename of the PCAP.
    :type filename: str
    :param data: The filedata of the PCAP.
    :type data: str
    :param source_name: The source which provided this PCAP.
    :type source_name: str,
                       :class:`crits.core.crits_mongoengine.EmbeddedSource`,
                       list of :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param user: The user adding the PCAP.
    :type user: str
    :param description: Description of the PCAP.
    :type description: str
    :param related_id: ObjectId of a top-level object related to this PCAP.
    :type related_id: str
    :param related_md5: MD5 of a top-level object related to this PCAP.
    :type related_md5: str
    :param related_type: The CRITs type of the related top-level object.
    :type related_type: str
    :param method: The method of acquiring this PCAP.
    :type method: str
    :param reference: A reference to the source of this PCAP.
    :type reference: str
    :param relationship: The relationship between the parent and the PCAP.
    :type relationship: str
    :param bucket_list: Bucket(s) to add to this PCAP.
    :type bucket_list: str(comma separated) or list.
    :param ticket: Ticket(s) to add to this PCAP.
    :type ticket: str(comma separated) or list.
    :returns: dict with keys:
              'success' (boolean),
              'message' (str),
              'md5' (str) if successful.
    """

    if not data:
        status = {'success': False, 'message': 'No data object passed in'}
        return status
    if len(data) <= 0:
        status = {'success': False, 'message': 'Data length <= 0'}
        return status
    if ((related_type and not (related_id or related_md5))
            or (not related_type and (related_id or related_md5))):
        status = {
            'success':
            False,
            'message':
            'Must specify both related_type and related_id or related_md5.'
        }
        return status

    if not source_name:
        return {"success": False, "message": "Missing source information."}

    related_obj = None
    if related_id or related_md5:
        if related_id:
            related_obj = class_from_id(related_type, related_id)
        else:
            related_obj = class_from_value(related_type, related_md5)
        if not related_obj:
            status = {'success': False, 'message': 'Related object not found.'}
            return status

    # generate md5 and timestamp
    md5 = hashlib.md5(data).hexdigest()
    timestamp = datetime.datetime.now()

    # generate PCAP
    is_pcap_new = False
    pcap = PCAP.objects(md5=md5).first()
    if not pcap:
        pcap = PCAP()
        pcap.filename = filename
        pcap.created = timestamp
        pcap.length = len(data)
        pcap.description = description
        pcap.md5 = md5
        is_pcap_new = True

    # generate source information and add to pcap
    if isinstance(source_name, basestring) and len(source_name) > 0:
        s = create_embedded_source(source_name,
                                   method=method,
                                   reference=reference,
                                   analyst=user)
        pcap.add_source(s)
    elif isinstance(source_name, EmbeddedSource):
        pcap.add_source(source_name, method=method, reference=reference)
    elif isinstance(source_name, list) and len(source_name) > 0:
        for s in source_name:
            if isinstance(s, EmbeddedSource):
                pcap.add_source(s, method=method, reference=reference)

    # add file to GridFS
    if not isinstance(pcap.filedata.grid_id, ObjectId):
        pcap.add_file_data(data)

    if bucket_list:
        pcap.add_bucket_list(bucket_list, user)

    if ticket:
        pcap.add_ticket(ticket, user)

    # save pcap
    pcap.save(username=user)

    # update relationship if a related top-level object is supplied
    if related_obj and pcap:
        if not relationship:
            relationship = "Related_To"
        pcap.add_relationship(rel_item=related_obj,
                              rel_type=relationship,
                              analyst=user,
                              get_rels=False)
        related_obj.save(username=user)
        pcap.save(username=user)

    # run pcap triage
    if is_pcap_new and data:
        pcap.reload()
        run_triage(pcap, user)

    status = {
        'success': True,
        'message': 'Uploaded pcap',
        'md5': md5,
        'id': str(pcap.id),
        'object': pcap
    }

    return status
Esempio n. 18
0
def get_pcap_details(md5, analyst):
    """
    Generate the data to render the PCAP details template.

    :param md5: The MD5 of the PCAP to get details for.
    :type md5: str
    :param analyst: The user requesting this information.
    :type analyst: str
    :returns: template (str), arguments (dict)
    """

    template = None
    sources = user_sources(analyst)
    pcap = PCAP.objects(md5=md5, source__name__in=sources).first()
    if not pcap:
        template = "error.html"
        args = {'error': 'PCAP not yet available or you do not have access to view it.'}
    else:

        pcap.sanitize("%s" % analyst)

        # remove pending notifications for user
        remove_user_from_notification("%s" % analyst, pcap.id, 'PCAP')

        # subscription
        subscription = {
                'type': 'PCAP',
                'id': pcap.id,
                'subscribed': is_user_subscribed("%s" % analyst,
                                                 'PCAP', pcap.id),
        }

        #objects
        objects = pcap.sort_objects()

        #relationships
        relationships = pcap.sort_relationships("%s" % analyst, meta=True)

        # relationship
        relationship = {
                'type': 'PCAP',
                'value': pcap.id
        }

        #comments
        comments = {'comments': pcap.get_comments(),
                    'url_key': md5}

        #screenshots
        screenshots = pcap.get_screenshots(analyst)

        # favorites
        favorite = is_user_favorite("%s" % analyst, 'PCAP', pcap.id)

        # services
        # Assume all PCAPs have the data available
        service_list = get_supported_services('PCAP')

        # analysis results
        service_results = pcap.get_analysis_results()

        args = {'service_list': service_list,
                'objects': objects,
                'relationships': relationships,
                'comments': comments,
                'favorite': favorite,
                'relationship': relationship,
                "subscription": subscription,
                "screenshots": screenshots,
                "service_results": service_results,
                "pcap": pcap}

    return template, args
Esempio n. 19
0
def class_from_id(type_, _id):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param _id: The ObjectId to search for.
    :type _id: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.core.crits_mongoengine import RelationshipType
    from crits.core.source_access import SourceAccess
    from crits.core.user_role import UserRole
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event, EventType
    from crits.indicators.indicator import Indicator, IndicatorAction
    from crits.ips.ip import IP
    from crits.objects.object_type import ObjectType
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData, RawDataType
    from crits.samples.backdoor import Backdoor
    from crits.samples.exploit import Exploit
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.targets.target import Target

    if not _id:
        return None

    # make sure it's a string
    _id = str(_id)

    if type_ == 'Backdoor':
        return Backdoor.objects(id=_id).first()
    if type_ == 'Campaign':
        return Campaign.objects(id=_id).first()
    elif type_ == 'Certificate':
        return Certificate.objects(id=_id).first()
    elif type_ == 'Comment':
        return Comment.objects(id=_id).first()
    elif type_ == 'Domain':
        return Domain.objects(id=_id).first()
    elif type_ == 'Email':
        return Email.objects(id=_id).first()
    elif type_ == 'Event':
        return Event.objects(id=_id).first()
    elif type_ == 'EventType':
        return EventType.objects(id=_id).first()
    elif type_ == 'Exploit':
        return Exploit.objects(id=_id).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=_id).first()
    elif type_ == 'IndicatorAction':
        return IndicatorAction.objects(id=_id).first()
    elif type_ == 'IP':
        return IP.objects(id=_id).first()
    elif type_ == 'ObjectType':
        return ObjectType.objects(id=_id).first()
    elif type_ == 'PCAP':
        return PCAP.objects(id=_id).first()
    elif type_ == 'RawData':
        return RawData.objects(id=_id).first()
    elif type_ == 'RawDataType':
        return RawDataType.objects(id=_id).first()
    elif type_ == 'RelationshipType':
        return RelationshipType.objects(id=_id).first()
    elif type_ == 'Sample':
        return Sample.objects(id=_id).first()
    elif type_ == 'SourceAccess':
        return SourceAccess.objects(id=_id).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=_id).first()
    elif type_ == 'Target':
        return Target.objects(id=_id).first()
    elif type_ == 'UserRole':
        return UserRole.objects(id=_id).first()
    else:
        return None
Esempio n. 20
0
def class_from_value(type_, value):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param value: The value to search for.
    :type value: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.actors.actor import ActorThreatIdentifier, Actor
    from crits.backdoors.backdoor import Backdoor
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.exploits.exploit import Exploit
    from crits.indicators.indicator import Indicator
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.targets.target import Target

    # Make sure value is a string...
    value = str(value)

    # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
    # the queries below will raise a ValidationError exception.
    if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
                  'Indicator', 'Screenshot'] and
       not ObjectId.is_valid(value.decode('utf8'))):
        return None

    if type_ == 'Actor':
        return Actor.objects(name=value).first()
    if type_ == 'Backdoor':
        return Backdoor.objects(id=value).first()
    elif type_ == 'ActorThreatIdentifier':
        return ActorThreatIdentifier.objects(name=value).first()
    elif type_ == 'Campaign':
        return Campaign.objects(name=value).first()
    elif type_ == 'Certificate':
        return Certificate.objects(md5=value).first()
    elif type_ == 'Comment':
        return Comment.objects(id=value).first()
    elif type_ == 'Domain':
        return Domain.objects(domain=value).first()
    elif type_ == 'Email':
        return Email.objects(id=value).first()
    elif type_ == 'Event':
        return Event.objects(id=value).first()
    elif type_ == 'Exploit':
        return Exploit.objects(id=value).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=value).first()
    elif type_ == 'IP':
        return IP.objects(ip=value).first()
    elif type_ == 'PCAP':
        return PCAP.objects(md5=value).first()
    elif type_ == 'RawData':
        return RawData.objects(md5=value).first()
    elif type_ == 'Sample':
        return Sample.objects(md5=value).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=value).first()
    elif type_ == 'Target':
        target = Target.objects(email_address=value).first()
        if target:
            return target
        else:
            return Target.objects(email_address__iexact=value).first()
    else:
        return None
Esempio n. 21
0
def class_from_id(type_, _id):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param _id: The ObjectId to search for.
    :type _id: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # doing this to avoid circular imports
    from crits.actors.actor import ActorThreatIdentifier, Actor
    from crits.backdoors.backdoor import Backdoor
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.core.source_access import SourceAccess
    from crits.core.user_role import UserRole
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.exploits.exploit import Exploit
    from crits.indicators.indicator import Indicator, IndicatorAction
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData, RawDataType
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.targets.target import Target

    if not _id:
        return None

    # make sure it's a string
    _id = str(_id)

    # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
    # the queries below will raise a ValidationError exception.
    if not ObjectId.is_valid(_id.decode('utf8')):
        return None

    if type_ == 'Actor':
        return Actor.objects(id=_id).first()
    elif type_ == 'Backdoor':
        return Backdoor.objects(id=_id).first()
    elif type_ == 'ActorThreatIdentifier':
        return ActorThreatIdentifier.objects(id=_id).first()
    elif type_ == 'Campaign':
        return Campaign.objects(id=_id).first()
    elif type_ == 'Certificate':
        return Certificate.objects(id=_id).first()
    elif type_ == 'Comment':
        return Comment.objects(id=_id).first()
    elif type_ == 'Domain':
        return Domain.objects(id=_id).first()
    elif type_ == 'Email':
        return Email.objects(id=_id).first()
    elif type_ == 'Event':
        return Event.objects(id=_id).first()
    elif type_ == 'Exploit':
        return Exploit.objects(id=_id).first()
    elif type_ == 'Indicator':
        return Indicator.objects(id=_id).first()
    elif type_ == 'IndicatorAction':
        return IndicatorAction.objects(id=_id).first()
    elif type_ == 'IP':
        return IP.objects(id=_id).first()
    elif type_ == 'PCAP':
        return PCAP.objects(id=_id).first()
    elif type_ == 'RawData':
        return RawData.objects(id=_id).first()
    elif type_ == 'RawDataType':
        return RawDataType.objects(id=_id).first()
    elif type_ == 'Sample':
        return Sample.objects(id=_id).first()
    elif type_ == 'SourceAccess':
        return SourceAccess.objects(id=_id).first()
    elif type_ == 'Screenshot':
        return Screenshot.objects(id=_id).first()
    elif type_ == 'Target':
        return Target.objects(id=_id).first()
    elif type_ == 'UserRole':
        return UserRole.objects(id=_id).first()
    else:
        return None
Esempio n. 22
0
def chopshop_carver(pcap_md5, options, analyst):
    # Make sure we can find ChopShop
    sc = get_config('ChopShop')
    user = get_user_info(analyst)

    if not sc:
        return {'success': False, 'message': 'Could not find ChopShop service.'}

    shop_path = "%s/shop" % str(sc['basedir'])
    if not os.path.exists(shop_path):
        return {'success': False, 'message': "ChopShop shop path does not exist."}

    sys.path.append(shop_path)
    import ChopLib as CL
    if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
        return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}

    # Until we have an smtp_extractor in ChopShop we have to resort to
    # to (ab)using payloads to dump the entire TCP stream and letting
    # handle_eml() process everything. We also use the payloads module
    # for handling raw carves. If a user wants to do SMTP and raw
    # simultaneously it won't work because we can't distinguish one
    # payloads module from another.
    if options.get('raw', False) and options.get('smtp', False):
        return {'success': False, 'message': "Can not process SMTP and raw simultaneously."}

    # Make sure we have a PCAP to work with
    pcap = PCAP.objects(md5=pcap_md5).first()
    if not pcap:
        return {'success': False, 'message': "No PCAP found."}
    pcap_data = pcap.filedata.read()
    if not pcap_data:
        return {'success': False, 'message': "Could not get PCAP from GridFS: %s" %  pcap_md5}

    source = pcap['source'][0]['name'] # XXX: This kind of sucks...

    # Create module string to pass to ChopShop
    modules = []
    if options.get('http_resp', False) or options.get('http_req', False):
        modules.append("http | http_extractor")

    if options.get('smtp', False) or options.get('raw', False):
        # ChopShop really needs an smtp_extractor, but there's no good
        # capability to do that yet. Maybe one day I'll build one. :)
        # For now, just use payloads and let handle_eml() sort it out.
        #
        # Raw carving works exactly the same way, just post-processed
        # differently.
        modules.append("payloads -b")

    if not modules:
        return {'success': False, 'message': "No modules specified."}

    mod_string = ';'.join(mod for mod in modules)

    from ChopLib import ChopLib
    from ChopUi import ChopUi

    choplib = ChopLib()
    chopui = ChopUi()

    choplib.base_dir = str(sc['basedir'])

    choplib.modules = mod_string

    chopui.jsonout = jsonhandler
    choplib.jsonout = True

    # ChopShop (because of pynids) needs to read a file off disk.
    # Write the pcap data to a temporary file.
    temp_pcap = tempfile.NamedTemporaryFile(delete=False)
    temp_pcap.write(pcap_data)
    temp_pcap.close()

    choplib.filename = temp_pcap.name
    chopui.bind(choplib)
    chopui.start()

    if chopui.jsonclass == None:
        os.unlink(temp_pcap.name)
        chopui.join()
        choplib.finish()
        choplib.join()
        return {'success': False,
                'message': 'Lost race condition in chopui. Try again.'}

    # ChopUI must be started before the jsonhandler class is insantiated.
    # Tell the class what we are looking for now that it exists.
    chopui.jsonclass.parse_options(options)

    choplib.start()

    while chopui.is_alive():
        time.sleep(.1)

    chopui.join()
    choplib.finish()
    choplib.join()

    os.unlink(temp_pcap.name)

    message = ''

    # Grab any carved HTTP bodies.
    for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
        if user.has_access_to(SampleACL.WRITE) and handle_file(name, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', md5_digest=md5_digest, related_type='PCAP'):
            # Specifically not using name here as I don't want to deal
            # with sanitizing it
            message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
        else:
            message += "Failed to save file %s." % md5_digest

    # Grab any carved SMTP returns.
    for blob in chopui.jsonclass.smtp_returns.values():
        ret = handle_eml(blob, source, None, analyst, 'ChopShop FileCarver',
                         related_id=pcap.id, related_type='PCAP',
                         relationship_type=RelationshipTypes.RELATED_TO)
        if not ret['status']:
            message += ret['reason']
            continue

        message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (reverse('crits-emails-views-email_detail', args=[ret['object'].id]), ret['object'].id, len(ret['attachments'].keys()))

        for md5_digest in ret['attachments'].keys():
            message += "<a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)

    # Handle raw returns.
    for id_, blob in chopui.jsonclass.raw_returns.items():
        if user.has_access_to(SampleACL.WRITE):
            md5_digest = handle_file(id_, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', related_type='PCAP')
        else:
            md5_digest = None
        if md5_digest:
            message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (id_, reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
        else:
            message += "Failed to save raw %s." % md5_digest

    # It's possible to have no files here if nothing matched.
    # Still return True as there were no problems.
    if not message:
        message = 'No files found.'
    return {'success': True, 'message': message}
Esempio n. 23
0
def pcap_pdml_html(pcap_md5, analyst):
    # check to see if there is a File object with the source reference of
    # 'tshark_pdml.html'. If there is, return it.
    # If not, generate it, save it, and return it.
    pcap = PCAP.objects(md5=pcap_md5).first()
    if not pcap:
        return "No PCAP found"
    else:
        coll = settings.COL_OBJECTS
        pdml_obj = None
        pdml_html = None
        for obj in pcap.obj:
            for source in obj.source:
                for instance in source.instances:
                    if instance.reference == 'tshark_pdml.html':
                        pdml_obj = obj
        if not pdml_obj:
            sc = get_config('MetaCap')
            tshark_bin = str(sc['tshark'])
            if not os.path.exists(tshark_bin):
                pdml_html = "Could not find tshark!"
                return {'html': pdml_html}

            pcap_data = pcap.filedata.read()
            if not pcap_data:
                pdml_html =  "Could not get PCAP from GridFS: %s" %  pcap_md5
                return {'html': pdml_html}

            # write PCAP to disk
            temp_pcap = tempfile.NamedTemporaryFile(delete=False)
            pcap_name = temp_pcap.name
            temp_pcap.write(pcap_data)
            temp_pcap.close()

            # use tshark to generate a pdml file
            temp_pdml = tempfile.NamedTemporaryFile(delete=False)
            args = [tshark_bin, "-n", "-r", pcap_name, "-T", "pdml"]
            tshark = Popen(args, stdout=temp_pdml, stderr=PIPE)
            tshark_out, tshark_err = tshark.communicate()
            if tshark.returncode != 0:
                return {'html': "%s, %s" % (tshark_out,tshark_err)}
            pdml_name = temp_pdml.name
            temp_pdml.seek(0)

            # transform PDML into HTML
            xsl_file = None
            for d in settings.SERVICE_DIRS:
                try:
                    file_dir = "%s/metacap_service" % d
                    xsl_file = open('%s/pdml2html.xsl' % file_dir, 'r')
                except IOError:
                    pass
            if not xsl_file:
                return {'html': 'Could not find XSL.'}

            parser = etree.XMLParser()
            parser.resolvers.add(FileResolver())
            save_pdml = False
            try:
                xml_input = etree.parse(temp_pdml, parser)
                xslt_root = etree.parse(xsl_file, parser)
                transform = etree.XSLT(xslt_root)
                pdml_html = str(transform(xml_input))
                save_pdml = True
            except Exception:
                return {'html': 'Could not parse/transform PDML output!'}

            temp_pdml.close()

            # delete PDML file
            os.unlink(pdml_name)
            os.unlink(pcap_name)

            #  save pdml_html as an object for this PCAP
            if save_pdml:
                fn = put_file_gridfs('tshark_pdml.html', pdml_html, collection=coll)
                if fn:
                    m = hashlib.md5()
                    m.update(pdml_html)
                    md5 = m.hexdigest()
                    pcap.add_object("Artifact",
                                    "File",
                                    md5,
                                    get_user_organization(analyst),
                                    "MetaCap",
                                    'tshark_pdml.html',
                                    analyst)
                    pcap.save()
        else:
            # get file from gridfs and return it
            obj_md5 = pdml_obj.value
            pdml_html = get_file_gridfs(obj_md5, collection=coll)
            if not pdml_html:
                return {'html': 'No file found in GridFS'}
        if not pdml_obj:
            pcap_objects = pcap.sort_objects()
            return {'html': pdml_html, 'objects': pcap_objects, 'id': pcap.id}
        else:
            return {'html': pdml_html}
Esempio n. 24
0
def get_pcap_details(md5, analyst):
    """
    Generate the data to render the PCAP details template.

    :param md5: The MD5 of the PCAP to get details for.
    :type md5: str
    :param analyst: The user requesting this information.
    :type analyst: str
    :returns: template (str), arguments (dict)
    """

    template = None
    sources = user_sources(analyst)
    pcap = PCAP.objects(md5=md5, source__name__in=sources).first()
    if not pcap:
        template = "error.html"
        args = {
            'error':
            'PCAP not yet available or you do not have access to view it.'
        }
    else:

        pcap.sanitize("%s" % analyst)

        # remove pending notifications for user
        remove_user_from_notification("%s" % analyst, pcap.id, 'PCAP')

        # subscription
        subscription = {
            'type': 'PCAP',
            'id': pcap.id,
            'subscribed': is_user_subscribed("%s" % analyst, 'PCAP', pcap.id),
        }

        #objects
        objects = pcap.sort_objects()

        #relationships
        relationships = pcap.sort_relationships("%s" % analyst, meta=True)

        # relationship
        relationship = {'type': 'PCAP', 'value': pcap.id}

        #comments
        comments = {'comments': pcap.get_comments(), 'url_key': md5}

        #screenshots
        screenshots = pcap.get_screenshots(analyst)

        # favorites
        favorite = is_user_favorite("%s" % analyst, 'PCAP', pcap.id)

        # services
        # Assume all PCAPs have the data available
        service_list = get_supported_services('PCAP')

        # analysis results
        service_results = pcap.get_analysis_results()

        args = {
            'service_list': service_list,
            'objects': objects,
            'relationships': relationships,
            'comments': comments,
            'favorite': favorite,
            'relationship': relationship,
            "subscription": subscription,
            "screenshots": screenshots,
            "service_results": service_results,
            "pcap": pcap
        }

    return template, args
Esempio n. 25
0
def pcap_tcpdump(pcap_md5, form, analyst):
    flag_list = []
    cleaned_data = form.cleaned_data

    # Make sure we can find tcpdump
    sc = get_config('MetaCap')
    tcpdump_bin = str(sc['tcpdump'])
    if not os.path.exists(tcpdump_bin):
        tcpdump_output = "Could not find tcpdump!"
        return tcpdump_output

    # Make sure we have a PCAP to work with
    pcap = PCAP.objects(md5=pcap_md5).first()
    if not pcap:
        return "No PCAP found"
    pcap_data = pcap.filedata.read()
    if not pcap_data:
        return "Could not get PCAP from GridFS: %s" %  pcap_md5

    # Use the filename if it's there, otherwise the md5.
    # This is used for the description of the carved sample.
    if pcap.filename:
        pcap_filename = pcap.filename
    else:
        pcap_filename = pcap_md5

    # Setup tcpdump arguments
    if cleaned_data['sequence']:
        flag_list.append("-S")
    if cleaned_data['timestamp']:
        flag_list.append("%s" % cleaned_data['timestamp'])
    if cleaned_data['verbose']:
        flag_list.append("%s" % cleaned_data['verbose'])
    if cleaned_data['data']:
        flag_list.append("%s" % cleaned_data['data'])
    # force -nN
    flag_list.append("-nN")
    # if we need to carve
    if cleaned_data['carve']:
        if not cleaned_data['bpf']:
            return "Must supply a BPF filter to carve."
        new_pcap = tempfile.NamedTemporaryFile(delete=False)
        flag_list.append("-w")
        flag_list.append(new_pcap.name)

    if cleaned_data['bpf']:
        flag_list.append('%s' % str(cleaned_data['bpf'].replace('"', '')))

    # write PCAP to disk
    # temp_out collects stdout and stderr
    # temp_pcap is the pcap to read
    # new_pcap is the pcap being written if carving
    temp_out = tempfile.NamedTemporaryFile(delete=False)
    temp_pcap = tempfile.NamedTemporaryFile(delete=False)
    pcap_name = temp_pcap.name
    temp_pcap.write(pcap_data)
    temp_pcap.close()
    args = [tcpdump_bin, '-r', temp_pcap.name] + flag_list
    tcpdump = Popen(args, stdout=temp_out, stderr=STDOUT)
    tcpdump.communicate()
    out_name = temp_out.name
    temp_out.seek(0)
    tcpdump_output = ''
    for line in iter(temp_out):
        tcpdump_output += "%s" % line
    temp_out.close()

    #delete temp files
    os.unlink(pcap_name)
    os.unlink(out_name)

    if cleaned_data['carve']:
        new_pcap_data = new_pcap.read()
        if len(new_pcap_data) > 24: # pcap-ng will change this.
            m = hashlib.md5()
            m.update(new_pcap_data)
            md5 = m.hexdigest()
            org = get_user_organization(analyst)
            result = handle_pcap_file("%s.pcap" % md5,
                                      new_pcap_data,
                                      org,
                                      user=analyst,
                                      description="%s of %s" % (cleaned_data['bpf'], pcap_filename),
                                      parent_id=pcap.id,
                                      parent_type="PCAP",
                                      method="MetaCap Tcpdumper")
            if result['success']:
                tcpdump_output = "<a href=\"%s\">View new pcap.</a>" % reverse('crits.pcaps.views.pcap_details', args=[result['md5']])
            else:
                tcpdump_output = result['message']
        else:
            tcpdump_output = "No packets matched the filter."

        os.unlink(new_pcap.name)

    return tcpdump_output
Esempio n. 26
0
def pcap_tcpdump(pcap_md5, form, analyst):
    flag_list = []
    cleaned_data = form.cleaned_data

    # Make sure we can find tcpdump
    sc = get_config("MetaCap")
    tcpdump_bin = str(sc["tcpdump"])
    if not os.path.exists(tcpdump_bin):
        tcpdump_output = "Could not find tcpdump!"
        return tcpdump_output

    # Make sure we have a PCAP to work with
    pcap = PCAP.objects(md5=pcap_md5).first()
    if not pcap:
        return "No PCAP found"
    pcap_data = pcap.filedata.read()
    if not pcap_data:
        return "Could not get PCAP from GridFS: %s" % pcap_md5

    # Use the filename if it's there, otherwise the md5.
    # This is used for the description of the carved sample.
    if pcap.filename:
        pcap_filename = pcap.filename
    else:
        pcap_filename = pcap_md5

    # Setup tcpdump arguments
    if cleaned_data["sequence"]:
        flag_list.append("-S")
    if cleaned_data["timestamp"]:
        flag_list.append("%s" % cleaned_data["timestamp"])
    if cleaned_data["verbose"]:
        flag_list.append("%s" % cleaned_data["verbose"])
    if cleaned_data["data"]:
        flag_list.append("%s" % cleaned_data["data"])
    # force -nN
    flag_list.append("-nN")
    # if we need to carve
    if cleaned_data["carve"]:
        if not cleaned_data["bpf"]:
            return "Must supply a BPF filter to carve."
        new_pcap = tempfile.NamedTemporaryFile(delete=False)
        flag_list.append("-w")
        flag_list.append(new_pcap.name)

    if cleaned_data["bpf"]:
        flag_list.append("%s" % str(cleaned_data["bpf"].replace('"', "")))

    # write PCAP to disk
    # temp_out collects stdout and stderr
    # temp_pcap is the pcap to read
    # new_pcap is the pcap being written if carving
    temp_out = tempfile.NamedTemporaryFile(delete=False)
    temp_pcap = tempfile.NamedTemporaryFile(delete=False)
    pcap_name = temp_pcap.name
    temp_pcap.write(pcap_data)
    temp_pcap.close()
    args = [tcpdump_bin, "-r", temp_pcap.name] + flag_list
    tcpdump = Popen(args, stdout=temp_out, stderr=STDOUT)
    tcpdump.communicate()
    out_name = temp_out.name
    temp_out.seek(0)
    tcpdump_output = ""
    for line in iter(temp_out):
        tcpdump_output += "%s" % line
    temp_out.close()

    # delete temp files
    os.unlink(pcap_name)
    os.unlink(out_name)

    if cleaned_data["carve"]:
        new_pcap_data = new_pcap.read()
        if len(new_pcap_data) > 24:  # pcap-ng will change this.
            m = hashlib.md5()
            m.update(new_pcap_data)
            md5 = m.hexdigest()
            org = get_user_organization(analyst)
            result = handle_pcap_file(
                "%s.pcap" % md5,
                new_pcap_data,
                org,
                user=analyst,
                description="%s of %s" % (cleaned_data["bpf"], pcap_filename),
                parent_id=pcap.id,
                parent_type="PCAP",
                method="MetaCap Tcpdumper",
            )
            if result["success"]:
                tcpdump_output = '<a href="%s">View new pcap.</a>' % reverse(
                    "crits.pcaps.views.pcap_details", args=[result["md5"]]
                )
            else:
                tcpdump_output = result["message"]
        else:
            tcpdump_output = "No packets matched the filter."

        os.unlink(new_pcap.name)

    return tcpdump_output
Esempio n. 27
0
def class_from_id(type_, _id):
    """
    Return an instantiated class object.

    :param type_: The CRITs top-level object type.
    :type type_: str
    :param _id: The ObjectId to search for.
    :type _id: str
    :returns: class which inherits from
              :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
    """

    # Quick fail
    if not _id or not type_:
        return None

    # doing this to avoid circular imports
    from crits.actors.actor import ActorThreatIdentifier, Actor
    from crits.backdoors.backdoor import Backdoor
    from crits.campaigns.campaign import Campaign
    from crits.certificates.certificate import Certificate
    from crits.comments.comment import Comment
    from crits.core.crits_mongoengine import Action
    from crits.core.source_access import SourceAccess
    from crits.core.user_role import UserRole
    from crits.domains.domain import Domain
    from crits.emails.email import Email
    from crits.events.event import Event
    from crits.exploits.exploit import Exploit
    from crits.indicators.indicator import Indicator
    from crits.ips.ip import IP
    from crits.pcaps.pcap import PCAP
    from crits.raw_data.raw_data import RawData, RawDataType
    from crits.samples.sample import Sample
    from crits.screenshots.screenshot import Screenshot
    from crits.signatures.signature import Signature, SignatureType, SignatureDependency
    from crits.targets.target import Target

    # make sure it's a string
    _id = str(_id)

    # Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
    # the queries below will raise a ValidationError exception.
    if not ObjectId.is_valid(_id.decode("utf8")):
        return None

    if type_ == "Actor":
        return Actor.objects(id=_id).first()
    elif type_ == "Backdoor":
        return Backdoor.objects(id=_id).first()
    elif type_ == "ActorThreatIdentifier":
        return ActorThreatIdentifier.objects(id=_id).first()
    elif type_ == "Campaign":
        return Campaign.objects(id=_id).first()
    elif type_ == "Certificate":
        return Certificate.objects(id=_id).first()
    elif type_ == "Comment":
        return Comment.objects(id=_id).first()
    elif type_ == "Domain":
        return Domain.objects(id=_id).first()
    elif type_ == "Email":
        return Email.objects(id=_id).first()
    elif type_ == "Event":
        return Event.objects(id=_id).first()
    elif type_ == "Exploit":
        return Exploit.objects(id=_id).first()
    elif type_ == "Indicator":
        return Indicator.objects(id=_id).first()
    elif type_ == "Action":
        return Action.objects(id=_id).first()
    elif type_ == "IP":
        return IP.objects(id=_id).first()
    elif type_ == "PCAP":
        return PCAP.objects(id=_id).first()
    elif type_ == "RawData":
        return RawData.objects(id=_id).first()
    elif type_ == "RawDataType":
        return RawDataType.objects(id=_id).first()
    elif type_ == "Sample":
        return Sample.objects(id=_id).first()
    elif type_ == "Signature":
        return Signature.objects(id=_id).first()
    elif type_ == "SignatureType":
        return SignatureType.objects(id=_id).first()
    elif type_ == "SignatureDependency":
        return SignatureDependency.objects(id=_id).first()
    elif type_ == "SourceAccess":
        return SourceAccess.objects(id=_id).first()
    elif type_ == "Screenshot":
        return Screenshot.objects(id=_id).first()
    elif type_ == "Target":
        return Target.objects(id=_id).first()
    elif type_ == "UserRole":
        return UserRole.objects(id=_id).first()
    else:
        return None