Esempio n. 1
0
    def _scan(self, context):
        logger.debug("Setting up shop...")
        shop_path = "%s/shop" % self.base_dir
        if not os.path.exists(self.base_dir):
            self._error("ChopShop path does not exist")
        elif not os.path.exists(shop_path):
            self._error("ChopShop shop path does not exist")
        else:
            sys.path.append(shop_path)
            import ChopLib as CL

            # I wanted to do this check in validate, but if it fails and
            # then you fix the path to point to the appropriate chopshop
            # it requires a webserver restart to take effect. So just do
            # the check at each scan.
            if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
                self._error("Need ChopShop 4.0 or newer")

            from ChopLib import ChopLib
            from ChopUi import ChopUi

            logger.debug("Scanning...")

            choplib = ChopLib()
            chopui = ChopUi()

            choplib.base_dir = self.base_dir

            # XXX: Convert from unicode to str...
            choplib.modules = str(self.modules)

            chopui.jsonout = jsonhandler
            choplib.jsonout = True

            # ChopShop (because of pynids) needs to read a file off disk.
            # The services framework forces you to use 'with' here. It's not
            # possible to just get a path to a file on disk.
            with self._write_to_file() as pcap_file:
                choplib.filename = pcap_file
                chopui.bind(choplib)
                chopui.start()
                chopui.jsonclass.set_service(self)
                choplib.start()

                while chopui.is_alive():
                    time.sleep(.1)

                chopui.join()
                choplib.finish()
                choplib.join()
Esempio n. 2
0
    def _scan(self, context):
        logger.debug("Setting up shop...")
        shop_path = "%s/shop" % self.base_dir
        if not os.path.exists(self.base_dir):
            self._error("ChopShop path does not exist")
        elif not os.path.exists(shop_path):
            self._error("ChopShop shop path does not exist")
        else:
            sys.path.append(shop_path)
            import ChopLib as CL

            # I wanted to do this check in validate, but if it fails and
            # then you fix the path to point to the appropriate chopshop
            # it requires a webserver restart to take effect. So just do
            # the check at each scan.
            if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
                self._error("Need ChopShop 4.0 or newer")

            from ChopLib import ChopLib
            from ChopUi import ChopUi

            logger.debug("Scanning...")

            choplib = ChopLib()
            chopui = ChopUi()

            choplib.base_dir = self.base_dir

            # XXX: Convert from unicode to str...
            choplib.modules = str(self.modules)

            chopui.jsonout = jsonhandler
            choplib.jsonout = True

            # ChopShop (because of pynids) needs to read a file off disk.
            # The services framework forces you to use 'with' here. It's not
            # possible to just get a path to a file on disk.
            with self._write_to_file() as pcap_file:
                choplib.filename = pcap_file
                chopui.bind(choplib)
                chopui.start()
                chopui.jsonclass.set_service(self)
                choplib.start()

                while chopui.is_alive():
                    time.sleep(.1)

                chopui.join()
                choplib.finish()
                choplib.join()
Esempio n. 3
0
    def run(self, obj, config):
        logger.debug("Setting up shop...")
        base_dir = config['basedir']
        shop_path = "%s/shop" % base_dir
        if not os.path.exists(base_dir):
            self._error("ChopShop path does not exist")
            return
        elif not os.path.exists(shop_path):
            self._error("ChopShop shop path does not exist")
            return

        sys.path.append(shop_path)
        from ChopLib import ChopLib
        from ChopUi import ChopUi

        logger.debug("Scanning...")

        choplib = ChopLib()
        chopui = ChopUi()

        choplib.base_dir = base_dir

        choplib.modules = "metacap -b"

        chopui.jsonout = jsonhandler
        choplib.jsonout = True

        # ChopShop (because of pynids) needs to read a file off disk.
        # The services framework forces you to use 'with' here. It's not
        # possible to just get a path to a file on disk.
        with self._write_to_file() as pcap_file:
            choplib.filename = pcap_file
            try:
                chopui.bind(choplib)
                chopui.start()
                while chopui.jsonclass == None:
                    time.sleep(.1)
                chopui.jsonclass.set_service(self)
                choplib.start()

                while chopui.is_alive():
                    time.sleep(.1)

            except Exception as e:
                self._error(str(e))
            finally:
                chopui.join()
                choplib.finish()
                choplib.join()
Esempio n. 4
0
    def run(self, obj, config):
        logger.debug("Setting up shop...")
        base_dir = config['basedir']
        shop_path = "%s/shop" % base_dir
        if not os.path.exists(base_dir):
            self._error("ChopShop path does not exist")
            return
        elif not os.path.exists(shop_path):
            self._error("ChopShop shop path does not exist")
            return

        sys.path.append(shop_path)
        from ChopLib import ChopLib
        from ChopUi import ChopUi

        logger.debug("Scanning...")

        choplib = ChopLib()
        chopui = ChopUi()

        choplib.base_dir = base_dir

        choplib.modules = "metacap -b"

        chopui.jsonout = jsonhandler
        choplib.jsonout = True

        # ChopShop (because of pynids) needs to read a file off disk.
        # The services framework forces you to use 'with' here. It's not
        # possible to just get a path to a file on disk.
        with self._write_to_file() as pcap_file:
            choplib.filename = pcap_file
            try:
                chopui.bind(choplib)
                chopui.start()
                while chopui.jsonclass == None:
                    time.sleep(.1)
                chopui.jsonclass.set_service(self)
                choplib.start()

                while chopui.is_alive():
                    time.sleep(.1)

            except Exception as e:
                self._error(str(e))
            finally:
                chopui.join()
                choplib.finish()
                choplib.join()
Esempio n. 5
0
    def _scan(self, context):
        logger.debug("Setting up shop...")
        shop_path = "%s/shop" % self.base_dir
        if not os.path.exists(self.base_dir):
            raise ServiceConfigError("ChopShop path does not exist")
        elif not os.path.exists(shop_path):
            raise ServiceConfigError("ChopShop shop path does not exist")
        else:
            sys.path.append(shop_path)
            from ChopLib import ChopLib
            from ChopUi import ChopUi

            logger.debug("Scanning...")

            choplib = ChopLib()
            chopui = ChopUi()

            choplib.base_dir = self.base_dir

            # XXX: Convert from unicode to str...
            choplib.modules = str(self.modules)

            chopui.jsonout = jsonhandler
            choplib.jsonout = True

            # ChopShop (because of pynids) needs to read a file off disk.
            # The services framework forces you to use 'with' here. It's not
            # possible to just get a path to a file on disk.
            with self._write_to_file() as pcap_file:
                choplib.filename = pcap_file
                chopui.bind(choplib)
                chopui.start()
                chopui.jsonclass.set_service(self)
                choplib.start()

                while chopui.is_alive():
                    time.sleep(.1)

                chopui.join()
                choplib.finish()
                choplib.join()
Esempio n. 6
0
    def run_module_info(self, modules):
        clib = ChopLib()
        clib.text = True
        clib.modules = modules
        clib.modinfo = True
        clib.start()

        stopped = False
        message_queue = clib.get_message_queue()

        while not stopped and clib.is_alive():
            try:
                message = message_queue.get(True, .1)
            except Queue.Empty, e:
                continue

            #clean up messages
            if message['type'] == 'ctrl':
                #self.send_message(message['data']['msg'] )
                if message['data']['msg'] == 'finished':
                    stopped = True
            elif message['type'] == 'text':
                self.send_message(message['data']['data'])
Esempio n. 7
0
    def run_module_info(self, modules):
        clib = ChopLib()
        clib.text = True
        clib.modules = modules
        clib.modinfo = True
        clib.start()

        stopped = False
        message_queue = clib.get_message_queue()

        while not stopped and clib.is_alive():
            try:
                message = message_queue.get(True, .1)
            except Queue.Empty, e:
                continue

            #clean up messages
            if message['type'] == 'ctrl':
                #self.send_message(message['data']['msg'] )
                if message['data']['msg'] == 'finished':
                    stopped = True
            elif message['type'] == 'text':
                self.send_message(message['data']['data'])
Esempio n. 8
0
def chopshop_carver(pcap_md5, options, analyst):
    # Make sure we can find ChopShop
    sc = get_config('ChopShop')
    user = get_user_info(analyst)

    if not sc:
        return {'success': False, 'message': 'Could not find ChopShop service.'}

    shop_path = "%s/shop" % str(sc['basedir'])
    if not os.path.exists(shop_path):
        return {'success': False, 'message': "ChopShop shop path does not exist."}

    sys.path.append(shop_path)
    import ChopLib as CL
    if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
        return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}

    # Until we have an smtp_extractor in ChopShop we have to resort to
    # to (ab)using payloads to dump the entire TCP stream and letting
    # handle_eml() process everything. We also use the payloads module
    # for handling raw carves. If a user wants to do SMTP and raw
    # simultaneously it won't work because we can't distinguish one
    # payloads module from another.
    if options.get('raw', False) and options.get('smtp', False):
        return {'success': False, 'message': "Can not process SMTP and raw simultaneously."}

    # Make sure we have a PCAP to work with
    pcap = PCAP.objects(md5=pcap_md5).first()
    if not pcap:
        return {'success': False, 'message': "No PCAP found."}
    pcap_data = pcap.filedata.read()
    if not pcap_data:
        return {'success': False, 'message': "Could not get PCAP from GridFS: %s" %  pcap_md5}

    source = pcap['source'][0]['name'] # XXX: This kind of sucks...

    # Create module string to pass to ChopShop
    modules = []
    if options.get('http_resp', False) or options.get('http_req', False):
        modules.append("http | http_extractor")

    if options.get('smtp', False) or options.get('raw', False):
        # ChopShop really needs an smtp_extractor, but there's no good
        # capability to do that yet. Maybe one day I'll build one. :)
        # For now, just use payloads and let handle_eml() sort it out.
        #
        # Raw carving works exactly the same way, just post-processed
        # differently.
        modules.append("payloads -b")

    if not modules:
        return {'success': False, 'message': "No modules specified."}

    mod_string = ';'.join(mod for mod in modules)

    from ChopLib import ChopLib
    from ChopUi import ChopUi

    choplib = ChopLib()
    chopui = ChopUi()

    choplib.base_dir = str(sc['basedir'])

    choplib.modules = mod_string

    chopui.jsonout = jsonhandler
    choplib.jsonout = True

    # ChopShop (because of pynids) needs to read a file off disk.
    # Write the pcap data to a temporary file.
    temp_pcap = tempfile.NamedTemporaryFile(delete=False)
    temp_pcap.write(pcap_data)
    temp_pcap.close()

    choplib.filename = temp_pcap.name
    chopui.bind(choplib)
    chopui.start()

    if chopui.jsonclass == None:
        os.unlink(temp_pcap.name)
        chopui.join()
        choplib.finish()
        choplib.join()
        return {'success': False,
                'message': 'Lost race condition in chopui. Try again.'}

    # ChopUI must be started before the jsonhandler class is insantiated.
    # Tell the class what we are looking for now that it exists.
    chopui.jsonclass.parse_options(options)

    choplib.start()

    while chopui.is_alive():
        time.sleep(.1)

    chopui.join()
    choplib.finish()
    choplib.join()

    os.unlink(temp_pcap.name)

    message = ''

    # Grab any carved HTTP bodies.
    for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
        if user.has_access_to(SampleACL.WRITE) and handle_file(name, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', md5_digest=md5_digest, related_type='PCAP'):
            # Specifically not using name here as I don't want to deal
            # with sanitizing it
            message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
        else:
            message += "Failed to save file %s." % md5_digest

    # Grab any carved SMTP returns.
    for blob in chopui.jsonclass.smtp_returns.values():
        ret = handle_eml(blob, source, None, analyst, 'ChopShop FileCarver',
                         related_id=pcap.id, related_type='PCAP',
                         relationship_type=RelationshipTypes.RELATED_TO)
        if not ret['status']:
            message += ret['reason']
            continue

        message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (reverse('crits-emails-views-email_detail', args=[ret['object'].id]), ret['object'].id, len(ret['attachments'].keys()))

        for md5_digest in ret['attachments'].keys():
            message += "<a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)

    # Handle raw returns.
    for id_, blob in chopui.jsonclass.raw_returns.items():
        if user.has_access_to(SampleACL.WRITE):
            md5_digest = handle_file(id_, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', related_type='PCAP')
        else:
            md5_digest = None
        if md5_digest:
            message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (id_, reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
        else:
            message += "Failed to save raw %s." % md5_digest

    # It's possible to have no files here if nothing matched.
    # Still return True as there were no problems.
    if not message:
        message = 'No files found.'
    return {'success': True, 'message': message}
Esempio n. 9
0
def chopshop_carver(pcap_md5, options, analyst):
    # Make sure we can find ChopShop
    sc = get_config('ChopShop')
    if not sc:
        return {
            'success': False,
            'message': 'Could not find ChopShop service.'
        }

    shop_path = "%s/shop" % str(sc['basedir'])
    if not os.path.exists(shop_path):
        return {
            'success': False,
            'message': "ChopShop shop path does not exist."
        }

    sys.path.append(shop_path)
    import ChopLib as CL
    if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
        return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}

    # Until we have an smtp_extractor in ChopShop we have to resort to
    # to (ab)using payloads to dump the entire TCP stream and letting
    # handle_eml() process everything. We also use the payloads module
    # for handling raw carves. If a user wants to do SMTP and raw
    # simultaneously it won't work because we can't distinguish one
    # payloads module from another.
    if options.get('raw', False) and options.get('smtp', False):
        return {
            'success': False,
            'message': "Can not process SMTP and raw simultaneously."
        }

    # Make sure we have a PCAP to work with
    pcap = PCAP.objects(md5=pcap_md5).first()
    if not pcap:
        return {'success': False, 'message': "No PCAP found."}
    pcap_data = pcap.filedata.read()
    if not pcap_data:
        return {
            'success': False,
            'message': "Could not get PCAP from GridFS: %s" % pcap_md5
        }

    source = pcap['source'][0]['name']  # XXX: This kind of sucks...

    # Create module string to pass to ChopShop
    modules = []
    if options.get('http_resp', False) or options.get('http_req', False):
        modules.append("http | http_extractor")

    if options.get('smtp', False) or options.get('raw', False):
        # ChopShop really needs an smtp_extractor, but there's no good
        # capability to do that yet. Maybe one day I'll build one. :)
        # For now, just use payloads and let handle_eml() sort it out.
        #
        # Raw carving works exactly the same way, just post-processed
        # differently.
        modules.append("payloads -b")

    if not modules:
        return {'success': False, 'message': "No modules specified."}

    mod_string = ';'.join(mod for mod in modules)

    from ChopLib import ChopLib
    from ChopUi import ChopUi

    choplib = ChopLib()
    chopui = ChopUi()

    choplib.base_dir = str(sc['basedir'])

    choplib.modules = mod_string

    chopui.jsonout = jsonhandler
    choplib.jsonout = True

    # ChopShop (because of pynids) needs to read a file off disk.
    # Write the pcap data to a temporary file.
    temp_pcap = tempfile.NamedTemporaryFile(delete=False)
    temp_pcap.write(pcap_data)
    temp_pcap.close()

    choplib.filename = temp_pcap.name
    chopui.bind(choplib)
    chopui.start()

    if chopui.jsonclass == None:
        os.unlink(temp_pcap.name)
        chopui.join()
        choplib.finish()
        choplib.join()
        return {
            'success': False,
            'message': 'Lost race condition in chopui. Try again.'
        }

    # ChopUI must be started before the jsonhandler class is insantiated.
    # Tell the class what we are looking for now that it exists.
    chopui.jsonclass.parse_options(options)

    choplib.start()

    while chopui.is_alive():
        time.sleep(.1)

    chopui.join()
    choplib.finish()
    choplib.join()

    os.unlink(temp_pcap.name)

    message = ''

    # Grab any carved HTTP bodies.
    for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
        if handle_file(name,
                       blob,
                       source,
                       related_md5=pcap_md5,
                       user=analyst,
                       method='ChopShop Filecarver',
                       md5_digest=md5_digest,
                       related_type='PCAP'):
            # Specifically not using name here as I don't want to deal
            # with sanitizing it
            message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (
                reverse('crits.samples.views.detail', args=[md5_digest
                                                            ]), md5_digest)
        else:
            message += "Failed to save file %s." % md5_digest

    # Grab any carved SMTP returns.
    for blob in chopui.jsonclass.smtp_returns.values():
        ret = handle_eml(blob,
                         source,
                         None,
                         analyst,
                         'ChopShop FileCarver',
                         related_id=pcap.id,
                         related_type='PCAP',
                         relationship_type=RelationshipTypes.RELATED_TO)
        if not ret['status']:
            message += ret['reason']
            continue

        message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (
            reverse('crits.emails.views.email_detail', args=[
                ret['object'].id
            ]), ret['object'].id, len(ret['attachments'].keys()))

        for md5_digest in ret['attachments'].keys():
            message += "<a href=\"%s\">%s</a><br />" % (reverse(
                'crits.samples.views.detail', args=[md5_digest]), md5_digest)

    # Handle raw returns.
    for id_, blob in chopui.jsonclass.raw_returns.items():
        md5_digest = handle_file(id_,
                                 blob,
                                 source,
                                 related_md5=pcap_md5,
                                 user=analyst,
                                 method='ChopShop Filecarver',
                                 related_type='PCAP')
        if md5_digest:
            message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (
                id_, reverse('crits.samples.views.detail',
                             args=[md5_digest]), md5_digest)
        else:
            message += "Failed to save raw %s." % md5_digest

    # It's possible to have no files here if nothing matched.
    # Still return True as there were no problems.
    if not message:
        message = 'No files found.'
    return {'success': True, 'message': message}
Esempio n. 10
0
    def run(self, obj, config):
        # When running under mod_wsgi we have to make sure sys.stdout is not
        # going to the real stdout. This is because multiprocessing (used by
        # choplib internally) does sys.stdout.flush(), which mod_wsgi doesn't
        # like. Work around by pointing sys.stdout somewhere that mod_wsgi
        # doesn't care about.
        sys.stdout = sys.stderr
        sys.stdin = open(os.devnull)

        logger.debug("Initializing ChopShop service.")
        basedir = config['basedir']
        modules = ""
        if 'HTTP' in config['modules']:
            modules += ";http | http_extractor -m"
        if 'DNS' in config['modules']:
            modules += ";dns | dns_extractor"

        logger.debug("Setting up shop...")
        shop_path = "%s/shop" % basedir
        if not os.path.exists(basedir):
            self._error("ChopShop path does not exist")
            return
        elif not os.path.exists(shop_path):
            self._error("ChopShop shop path does not exist")
            return
        sys.path.append(shop_path)
        import ChopLib as CL

        # I wanted to do this check in validate, but if it fails and
        # then you fix the path to point to the appropriate chopshop
        # it requires a webserver restart to take effect. So just do
        # the check at each scan.
        if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
            self._error("Need ChopShop 4.0 or newer")

        from ChopLib import ChopLib
        from ChopUi import ChopUi

        logger.debug("Scanning...")

        choplib = ChopLib()
        chopui = ChopUi()

        choplib.base_dir = basedir

        # XXX: Convert from unicode to str...
        choplib.modules = str(modules)

        chopui.jsonout = jsonhandler
        choplib.jsonout = True

        # ChopShop (because of pynids) needs to read a file off disk.
        # The services framework forces you to use 'with' here. It's not
        # possible to just get a path to a file on disk.
        with self._write_to_file() as pcap_file:
            choplib.filename = pcap_file
            chopui.bind(choplib)
            chopui.start()
            chopui.jsonclass.set_service(self)
            choplib.start()

            while chopui.is_alive():
                time.sleep(.1)

            chopui.join()
            choplib.finish()
            choplib.join()
Esempio n. 11
0
    def run(self, obj, config):
        # When running under mod_wsgi we have to make sure sys.stdout is not
        # going to the real stdout. This is because multiprocessing (used by
        # choplib internally) does sys.stdout.flush(), which mod_wsgi doesn't
        # like. Work around by pointing sys.stdout somewhere that mod_wsgi
        # doesn't care about.
        sys.stdout = sys.stderr
        sys.stdin = open(os.devnull)

        logger.debug("Initializing ChopShop service.")
        basedir = config['basedir']
        modules = ""
        if 'HTTP' in config['modules']:
            modules += ";http | http_extractor -m"
        if 'DNS' in config['modules']:
            modules += ";dns | dns_extractor"

        logger.debug("Setting up shop...")
        shop_path = "%s/shop" % basedir
        if not os.path.exists(basedir):
            self._error("ChopShop path does not exist")
            return
        elif not os.path.exists(shop_path):
            self._error("ChopShop shop path does not exist")
            return
        sys.path.append(shop_path)
        import ChopLib as CL

        # I wanted to do this check in validate, but if it fails and
        # then you fix the path to point to the appropriate chopshop
        # it requires a webserver restart to take effect. So just do
        # the check at each scan.
        if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
            self._error("Need ChopShop 4.0 or newer")

        from ChopLib import ChopLib
        from ChopUi import ChopUi

        logger.debug("Scanning...")

        choplib = ChopLib()
        chopui = ChopUi()

        choplib.base_dir = basedir

        # XXX: Convert from unicode to str...
        choplib.modules = str(modules)

        chopui.jsonout = jsonhandler
        choplib.jsonout = True

        # ChopShop (because of pynids) needs to read a file off disk.
        # The services framework forces you to use 'with' here. It's not
        # possible to just get a path to a file on disk.
        with self._write_to_file() as pcap_file:
            choplib.filename = pcap_file
            chopui.bind(choplib)
            chopui.start()
            chopui.jsonclass.set_service(self)
            choplib.start()

            while chopui.is_alive():
                time.sleep(.1)

            chopui.join()
            choplib.finish()
            choplib.join()