Beispiel #1
0
    def _run(self, scanObject, result, depth, args):
        '''Laika framework module logic execution'''
        moduleResult = [] 

        # Determine file limit from arguments
        file_limit = int(get_option(args, 'filelimit', 'zipfilelimit', 0))
        byte_limit = int(get_option(args, 'bytelimit', 'zipbytelimit', 0))
        zippw = get_option(args, 'password', 'zippassword', '')

        # write temporary file so we can open it with zipfile
        file = cStringIO.StringIO()
        file.write(scanObject.buffer)

        try:
            logging.debug("first attempt at unzipping..")
            self._unzip_file(self, moduleResult, file, scanObject, result, zippw, file_limit, byte_limit)
        except zipfile.BadZipfile:
            try:
                # try to repair the zip file (known python limitation)
                logging.debug("error extracting zip, trying to fix it")
                self._fix_bad_zip(file, scanObject.buffer)
                self._unzip_file(self, moduleResult, file, scanObject, result, zippw, file_limit, byte_limit)
            #except QuitScanException:
            #    raise
            except:
                # add a flag to the object to indicate it couldn't be extracted
                logging.debug("couldn't fix zip, marking it as corrupt")
                scanObject.addFlag("CORRUPT_ZIP")
                # error logging handled by SI_MODULE wrapper
                raise
        finally:
            scanObject.addMetadata(self.module_name, "Unzipped", len(moduleResult))
            file.close()
        return moduleResult
Beispiel #2
0
    def _run(self, scanObject, result, depth, args):
        """Laika framework _module logic execution"""
        moduleResult = [] 

        # Determine file limit from arguments
        file_limit = int(get_option(args, 'filelimit', 'zipfilelimit', 0))
        byte_limit = int(get_option(args, 'bytelimit', 'zipbytelimit', 0))
        zippw = get_option(args, 'password', 'zippassword', '')

        # write temporary file so we can open it with zipfile
        file = cStringIO.StringIO()
        file.write(scanObject.buffer)

        try:
            logging.debug("first attempt at unzipping..")
            self._unzip_file(self, moduleResult, file, scanObject, result, zippw, file_limit, byte_limit)
        except zipfile.BadZipfile:
            try:
                # try to repair the zip file (known python limitation)
                logging.debug("error extracting zip, trying to fix it")
                self._fix_bad_zip(file, scanObject.buffer)
                self._unzip_file(self, moduleResult, file, scanObject, result, zippw, file_limit, byte_limit)
            #except QuitScanException:
            #    raise
            except:
                # add a flag to the object to indicate it couldn't be extracted
                logging.debug("couldn't fix zip, marking it as corrupt")
                scanObject.addFlag("CORRUPT_ZIP")
                # error logging handled by SI_MODULE wrapper
                raise
        finally:
            scanObject.addMetadata(self.module_name, "Unzipped", len(moduleResult))
            file.close()
        return moduleResult
Beispiel #3
0
    def _run(self, scanObject, result, depth, args):
        '''
        Arguments:
        unix_socket     -- Path to the clamd unix socket (str)
        max_bytes       -- Maximum number of bytes to scan (0 is unlimited) (int)
        timeout         -- Max number of seconds to scan (float)

        Returns:
        Flags           -- Virus name hits
        Metadata        -- Unix socket or daemon errors
        '''
        moduleResult = []

        # Defualt max of 20 MB
        default_max_bytes = 20000000
        # Default timeout of 10.0 seconds
        default_timeout = 10.0
        # Default clamd unix socket
        default_unix_socket = '/var/run/clamav/clamd.sock'

        unix_socket = str(get_option(args, 'unixsocket', 'scanclamavunixsocket', default_unix_socket))
        max_bytes = int(get_option(args, 'maxbytes', 'scanclamavmaxbytes', default_max_bytes))
        timeout = float(get_option(args, 'timeout', 'scanclamavtimeout', default_timeout))

        if timeout < 0.01:
            timeout = default_timeout

        # Connect to daemon
        try:
            clam = pyclamd.ClamdUnixSocket(filename=unix_socket, timeout=timeout)
        except IOError:
            logging.debug('IOError: Cannot connect to clamd unix socket file')
            scanObject.addMetadata(self.module_name, 'Error', 'IOError: clamd socket')
            return moduleResult

        errmsg = None
        try:
            # Scan the buffer with clamav
            if max_bytes <= 0:
                clam_result = clam.scan_stream(scanObject.buffer)
            else:
                clam_result = clam.scan_stream(str(buffer(scanObject.buffer, 0, max_bytes)))

            # Process a result
            if clam_result:
                status, virusname = clam_result['stream']
                scanObject.addFlag("%s:%s" % (self.flag_prefix, str(virusname)))
        except ValueError as e:
            errmsg = "ValueError (BufferTooLong): %s" % e
        except IOError as e:
            errmsg = "IOError (ScanError): %s" % e
        except Exception as e:
            errmsg = "Unexpected error: %s" % e

        if errmsg:
            logging.debug(errmsg)
            scanObject.addMetadata(self.module_name, 'Error', errmsg)

        return moduleResult
Beispiel #4
0
    def _run(self, scanObject, result, depth, args):
        '''
        Assumes:
            there is a string like object in scanObject.buffer
        Ensures:
            hash values added using scanObject.addMetadata

        Laika Config File Options:
            hashmd5:    "1" = md5.hexdigest,    "0" = omit
            hashSHA1:   "1" = sha1.hexdigest,   "0" = omit
            hashSHA256: "1" = sha256.hexdigest, "0" = omit
            hashSHA512: "1" = sha256.hexdigest, "0" = omit
            hashSHA1:   "1" = sha1.hexdigest,   "0" = omit
            ssdeep:     "1" = ssdeep.hash,      "0" = omit

        Function Arguments:
        :param scanObject:<laikaboss.objectmodel.ScanObject>
        :param result:<laikaboss.objectmodel.ScanResult>
        :param depth:<int>
        :param args:<dict> --execution flow controls--
                    Valid args names <str> (value must be 1, 0, "1", or "0")
                        1/"1": Generate the hash of named type
                        0/"0": Omit the hash of named type
                        default args:
                        {"md5":1,
                         "SHA1":0,
                         "SHA256":1,
                         "SHA512":1,
                         "ssdeep":0}

        :return: Always returns a empty list (no child objects)
        '''
        moduleResult = []
        metaDict = {}
        if int(get_option(args, 'md5', 'hashmd5', "md5" in self.module_defaults)):
            metaDict['md5'] = hashlib.md5(scanObject.buffer).hexdigest()
        if int(get_option(args, 'SHA1', 'hashSHA1', "SHA1" in self.module_defaults)):
            metaDict['SHA1'] = hashlib.sha1(scanObject.buffer).hexdigest()
        if int(get_option(args, 'SHA256', 'hashSHA256', "SHA256" in self.module_defaults)):
            metaDict['SHA256'] = hashlib.sha256(scanObject.buffer).hexdigest()
        if int(get_option(args, 'SHA512', 'hashSHA512', "SHA512" in self.module_defaults)):
            metaDict['SHA512'] = hashlib.sha512(scanObject.buffer).hexdigest()
        if int(get_option(args, 'ssdeep', 'hashssdeep', "ssdeep" in self.module_defaults)):
            #only import ssdeep if dispatched.
            #Prevents import error if you don't have/want the package
            #python should keep handing you the original, minimal/no overhead
            try:
                import ssdeep
                metaDict['ssdeep'] = ssdeep.hash(scanObject.buffer)
            except ImportError:
                metaDict['ssdeep'] = "" #indicate ssdeep was configured but failed


        scanObject.addMetadata(self.module_name, "HASHES", metaDict)
        
        return moduleResult
Beispiel #5
0
    def _run(self, scanObject, result, depth, args):
        """
        Arguments:
        unixsocket     -- Path to the clamd unix socket (str)
        maxbytes       -- Maximum number of bytes to scan (0 is unlimited) (int)

        Returns:
        Flags           -- Virus name hits
        Metadata        -- Unix socket or daemon errors
        """
        moduleResult = []

        unix_socket = str(
            get_option(args, 'unixsocket', 'scanclamavunixsocket',
                       '/var/run/clamav/clamd.ctl'))
        max_bytes = int(
            get_option(args, 'maxbytes', 'scanclamavmaxbytes', 20000000))

        # Connect to daemon
        if not self.clam:
            try:
                self.clam = pyclamd.ClamdUnixSocket(filename=unix_socket)
            except IOError:
                logging.debug(
                    'IOError: Cannot connect to clamd unix socket file')
                scanObject.addMetadata(self.module_name, 'Error',
                                       'IOError: clamd socket')
                raise

        try:
            # Scan the buffer with clamav
            if max_bytes <= 0:
                clam_result = self.clam.scan_stream(scanObject.buffer)
            else:
                clam_result = self.clam.scan_stream(
                    str(buffer(scanObject.buffer, 0, max_bytes)))

            # Process a result
            if clam_result:
                status, virusname = clam_result['stream']
                scanObject.addFlag("%s:%s" %
                                   (self.flag_prefix, str(virusname)))
        except ValueError as e:
            scanObject.addMetadata(self.module_name, 'Error',
                                   'ValueError (BufferTooLong): %s' % str(e))
        except IOError as e:
            scanObject.addMetadata(self.module_name, 'Error',
                                   'IOError (ScanError): %s' % str(e))

        return moduleResult
    def _run(self, scanObject, result, depth, args):
        '''
        Here is where the actual magic happens. This method is called on the object being scanned given that the dispatcher has a matching rule triggering this module.

        There are four types of actions that modules do on scan objects:
          * Add flags.
          * Add metadata.
          * Explode children.
          * Interact with external systems.

        Any module can perform as many of these actions as necessary, and many do more than one, such as adding flags and metadata.

        The parameters provided to this method by the framework are:
          * scanObject: This is the object currently being scanned (duh) of type ScanObject. It contains all of the flags, metadata and other assigned fields regarding this spefic instance of the object. This parameter is used by modules to access the buffer, add flags and add metadata.
          * result: This is the umbrella object that contains all of the ScanObjects created during the root object's scan. It is primarily used by modules to access the parent object of the scanObject when needed.
          * depth: This is a leftover parameter from the tracking of the depth of recursion of objects. It is recommended to get this value from the scanObject itself.
          * args: This is a dictionary of the runtime arguments provided to the module by the dispatcher. This parameter provides customization of module runs from the dispatcher, so that the module can operate differently based on the type/context of the scanObject.

        This method MUST return a list of ModuleObjects. ModuleObjects represent the children exploded (or the less violent extracted) from the scanObject by this module. If the module does not explode any children (as most do not), simply return an empty list. Not returning a list causes the framework to log an error for this module each time it is run, but will not prevent it from running next time, nor will it remove any flags/metadata added by the module run.
        '''
        # This variable is recommended to be used by all modules as the returned list of ModuleObjects, populated as the children objects are found.
        moduleResult = [] 

        # A typical first step is define the configuration options of the module.
        # A best practice for configuration options to a module is to honor them in this precedence:
        #   3. value set as default in this code
        #   2. value specified in config file
        #   1. value specified in arguments of the invocation of this module (via the dispatcher)
        # To help with this, the get_option(...) method provided in the laikaboss.util module provides a single method call to set the option according to this precedence.
        helloworld_param = int(get_option(args, 'param', 'helloworldparam', 10))

        # To add flags to the object, use the addFlag method of the scanObject.
        # Flags should have the following three parts, separated by ':'s:
        #   * Shortened name of the module.
        #   * 'nfo' if the flag is informational, 'err' if the flag is for a policy/logic error (versus a programatic error), or leave blank if a typical flag.
        #   * Expressive name representing the atomic concept of the flag.
        scanObject.addFlag('e_helloworld:nfo:justsayinghello')

        # To add metadata to the object, use the addMetadata method of the scanObject.
        scanObject.addMetadata(self.module_name, "minsize", helloworld_param)

        # If you want to call a separate function, pass the data and let the 
        # function set flags on the data. The function will also modify the moduleResult variable
        # to add subobjects 
        flags = self._helloworld(scanObject.buffer, moduleResult, helloworld_param)

        for flag in flags:
            scanObject.addFlag(flag)

        # Whenever you need to do a try-except, you must make sure to catch and raise the framework ScanError exceptions.
        try:
            nonexistant_var.bad_method_call()
        except NameError:
            pass
        except ScanError:
            raise

        # Always return a list of ModuleObjects (or empty list if no children)
        return moduleResult
Beispiel #7
0
    def _run(self, scanObject, result, depth, args):
        """
        Arguments:
        unixsocket     -- Path to the clamd unix socket (str)
        maxbytes       -- Maximum number of bytes to scan (0 is unlimited) (int)

        Returns:
        Flags           -- Virus name hits
        Metadata        -- Unix socket or daemon errors
        """
        moduleResult = []

        unix_socket = str(get_option(args, "unixsocket", "scanclamavunixsocket", "/var/run/clamav/clamd.ctl"))
        max_bytes = int(get_option(args, "maxbytes", "scanclamavmaxbytes", 20000000))

        # Connect to daemon
        if not self.clam:
            try:
                self.clam = pyclamd.ClamdUnixSocket(filename=unix_socket)
            except IOError:
                logging.debug("IOError: Cannot connect to clamd unix socket file")
                scanObject.addMetadata(self.module_name, "Error", "IOError: clamd socket")
                raise

        try:
            # Scan the buffer with clamav
            if max_bytes <= 0:
                clam_result = self.clam.scan_stream(scanObject.buffer)
            else:
                clam_result = self.clam.scan_stream(str(buffer(scanObject.buffer, 0, max_bytes)))

            # Process a result
            if clam_result:
                status, virusname = clam_result["stream"]
                scanObject.addFlag("%s:%s" % (self.flag_prefix, str(virusname)))
        except ValueError as e:
            scanObject.addMetadata(self.module_name, "Error", "ValueError (BufferTooLong): %s" % str(e))
        except IOError as e:
            scanObject.addMetadata(self.module_name, "Error", "IOError (ScanError): %s" % str(e))

        return moduleResult
Beispiel #8
0
    def _run(self, scanObject, result, depth, args):
        moduleResult = []
        e = email.message_from_string(scanObject.buffer)

        attachments = []

        i = 1
        for p in e.walk():
            childBuffer = p.get_payload(decode=True)
            if childBuffer is not None:
                filename = p.get_filename()
                if filename is None:
                    filename = 'e_email_%s_%s' % (p.get_content_type(), i)
                else:
                    attachments.append(filename)
                logging.debug("explode email: found filename: %s" % (filename))
                moduleResult.append(
                    ModuleObject(buffer=childBuffer,
                                 externalVars=ExternalVars(
                                     filename=filename,
                                     contentType=p.get_content_maintype())))
                i += 1

        # If enabled, this will combine the email headers and all decoded
        # text portions contained in the email into a single object
        if strtobool(
                get_option(args, 'createhybrid', 'explodeemlcreatehybrid',
                           'False')):
            # First, grab the headers
            header_end = scanObject.buffer.find('\x0a\x0a')
            hybrid = scanObject.buffer[:header_end] + '\n\n'
            for mo in moduleResult:
                if 'text' in mo.externalVars.contentType:
                    hybrid += mo.buffer + '\n\n'

            # Add the hybrid as another object with a special content type
            # for easy identification.
            moduleResult.append(
                ModuleObject(
                    buffer=hybrid,
                    externalVars=ExternalVars(
                        filename='e_email_hybrid',
                        contentType='application/x-laika-eml-hybrid')))

        # Since we already gathered up the attachment names, we'll add them
        # on behalf of META_EMAIL
        scanObject.addMetadata('META_EMAIL', 'Attachments', attachments)

        return moduleResult
Beispiel #9
0
    def _run(self, scanObject, result, depth, args):
        moduleResult = [] 
        e = email.message_from_string(scanObject.buffer)

        attachments = []

        i = 1
        for p in e.walk():
            childBuffer = p.get_payload(decode=True)
            if childBuffer is not None:
                filename = p.get_filename() 
                if filename is None:
                    filename = 'e_email_%s_%s' % (p.get_content_type(),i)
                else:
                    attachments.append(filename)
                logging.debug("explode email: found filename: %s" % (filename))
                moduleResult.append(ModuleObject(buffer=childBuffer, 
                                                 externalVars=ExternalVars(filename=filename,
                                                                           contentType=p.get_content_maintype())))
                i += 1

        # If enabled, this will combine the email headers and all decoded
        # text portions contained in the email into a single object
        if strtobool(get_option(args, 'createhybrid', 'explodeemlcreatehybrid', 'False')):
            # First, grab the headers
            header_end = scanObject.buffer.find('\x0a\x0a')
            hybrid = scanObject.buffer[:header_end] + '\n\n'
            for mo in moduleResult:
                if 'text' in mo.externalVars.contentType:
                    hybrid += mo.buffer + '\n\n'

            # Add the hybrid as another object with a special content type
            # for easy identification.
            moduleResult.append(ModuleObject(buffer=hybrid, 
                                             externalVars=ExternalVars(filename='e_email_hybrid',
                                                                       contentType='application/x-laika-eml-hybrid')))
            
        # Since we already gathered up the attachment names, we'll add them
        # on behalf of META_EMAIL
        scanObject.addMetadata('META_EMAIL', 'Attachments', attachments)
 
        return moduleResult
Beispiel #10
0
    def _run(self, scanObject, result, depth, args):
        """Main module execution. Logs the scan result to fluentd."""
        tag  = get_option(args, "tag",  "fluent_tag",  "laikaboss.log_fluent")
        host = get_option(args, "host", "fluent_host", "localhost")
        port = int(get_option(args, "port", "fluent_port", 24224))
        bufmax = int(get_option(args, "bufmax", "fluent_bufmax", 1048576)) # 1 MB
        timeout = float(get_option(args, "timeout", "fluent_timeout", 3.0))
        label = get_option(args, "label", "fluent_label", "scan_result")

        sender = self._get_sender(tag, host, port, bufmax, timeout)
        log_record = self._parse_log_record(result)
        sender.emit(label, log_record)
        if sender.pendings: # buffer in case of failed transmission
            log_module_error(self.module_name, scanObject, result,
                'Log event failed emition. Will retry on next emition.')
        return []
Beispiel #11
0
    def _run(self, scanObject, result, depth, args):
        """Main _module execution. Logs the scan result to fluentd."""
        tag = get_option(args, "tag", "fluent_tag", "laikaboss.log_fluent")
        host = get_option(args, "host", "fluent_host", "localhost")
        port = int(get_option(args, "port", "fluent_port", 24224))
        bufmax = int(get_option(args, "bufmax", "fluent_bufmax",
                                1048576))  # 1 MB
        timeout = float(get_option(args, "timeout", "fluent_timeout", 3.0))
        label = get_option(args, "label", "fluent_label", "scan_result")

        sender = self._get_sender(tag, host, port, bufmax, timeout)
        log_record = self._parse_log_record(result)
        sender.emit(label, log_record)
        if sender.pendings:  # buffer in case of failed transmission
            log_module_error(
                self.module_name, scanObject, result,
                'Log event failed emition. Will retry on next emition.')
        return []
Beispiel #12
0
    def _run(self, scanObject, result, depth, args):
        '''Laika framework module logic execution'''
        moduleResult = []

        file_limit = int(get_option(args, 'filelimit', 'rarfilelimit', 0))
        byte_limit = int(get_option(args, 'bytelimit', 'rarbytelimit', 0))
        password = get_option(args, 'password', 'rarpassword')
        attempt_decrypt = strtobool(
            get_option(args, 'attemptdecrypt', 'rarattemptdecrypt', 'false'))
        temp_dir = get_option(args, 'tempdir', 'tempdir', '/tmp/laikaboss_tmp')
        if not os.path.isdir(temp_dir):
            os.mkdir(temp_dir)
            os.chmod(temp_dir, 0777)

        # A temp file must be created as UnRAR2 does not accept buffers
        with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:
            temp_file.write(scanObject.buffer)
            temp_file.flush()

            # RAR can be password protected, which encrypts the headers
            headers_are_encrypted = False
            # RAR can encrypt the files while leaving the headers decrypted
            files_are_encrypted = False

            rar = None
            # list of the file info objects
            infos = []

            try:
                logging.debug('%s: Attempting to open rar file',
                              self.module_name)
                # If headers are encrypted, the following will raise IncorrectRARPassword
                rar = UnRAR2.RarFile(temp_file.name)
                infos = rar.infolist()
                logging.debug('%s: Succeeded opening rar file',
                              self.module_name)

                # If files are encrypted, the filename will be prefixed with a '*'
                for info in infos:
                    if info.filename.startswith('*'):
                        logging.debug('%s: Rar files are encrypted',
                                      self.module_name)
                        scanObject.addFlag('ENCRYPTED_RAR')
                        scanObject.addMetadata(self.module_name, "Encrypted",
                                               "Protected Files")
                        files_are_encrypted = True
                        break
            except IncorrectRARPassword:
                logging.debug('%s: Rar headers are encrypted',
                              self.module_name)
                scanObject.addFlag('ENCRYPTED_RAR')
                scanObject.addMetadata(self.module_name, "Encrypted",
                                       "Protected Header")
                headers_are_encrypted = True
            except InvalidRARArchive:
                logging.debug('%s: Invalid Rar file')

            if (headers_are_encrypted
                    or files_are_encrypted) and attempt_decrypt:
                logging.debug('%s: Attempting to decrypt', self.module_name)
                possible_passwords = []

                # Passwords are sometimes sent in the email content. Use the content of the parent
                # object as the list of possible passwords
                parent_object = getParentObject(result, scanObject)
                if parent_object:
                    possible_passwords = _create_word_list(
                        parent_object.buffer)

                if password:
                    possible_passwords.insert(0, password)

                explode_temp_dir = os.path.join(temp_dir, 'exploderar')
                for possible_password in possible_passwords:
                    try:
                        logging.debug("EXPLODE_RAR: Attempting password '%s'",
                                      possible_password)
                        rar = UnRAR2.RarFile(temp_file.name,
                                             password=possible_password)
                        # Extraction is needed to force the exception on encrypted files
                        if files_are_encrypted:
                            rar.extract(path=explode_temp_dir)
                        infos = rar.infolist()
                        logging.debug("EXPLODE_RAR: Found password '%s'",
                                      possible_password)
                        scanObject.addFlag('rar:decrypted')
                        scanObject.addMetadata(self.module_name, 'Password',
                                               possible_password)
                        break
                    except IncorrectRARPassword:
                        continue
                if os.path.exists(explode_temp_dir):
                    remove_dir(explode_temp_dir)

            scanObject.addMetadata(self.module_name, "Total_Files", len(infos))
            file_count = 0
            exceeded_byte_limit = False
            for info in infos:
                if byte_limit and info.size > byte_limit:
                    logging.debug(
                        "EXPLODE_RAR: skipping file due to byte limit")
                    exceeded_byte_limit = True
                    continue
                try:
                    content = rar.read_files(info.filename)[0][1]
                    if byte_limit and len(content) > byte_limit:
                        logging.debug(
                            "EXPLODE_RAR: skipping file due to byte limit")
                        exceeded_byte_limit = True
                        continue
                    moduleResult.append(
                        ModuleObject(
                            buffer=content,
                            externalVars=ExternalVars(filename=info.filename)))
                except IndexError:
                    pass
                file_count += 1
                if file_limit and file_count >= file_limit:
                    scanObject.addFlag("rar:err:LIMIT_EXCEEDED")
                    logging.debug("EXPLODE_RAR: breaking due to file limit")
                    break
            if exceeded_byte_limit:
                scanObject.addFlag("rar:err:BYTE_LIMIT_EXCEEDED")

        scanObject.addMetadata(self.module_name, "Unzipped", len(moduleResult))
        return moduleResult
Beispiel #13
0
    def _run(self, scanObject, result, depth, args):
        '''Laika framework module logic execution'''
        moduleResult = []

        file_limit = int(get_option(args, 'filelimit', 'rarfilelimit', 0))
        byte_limit = int(get_option(args, 'bytelimit', 'rarbytelimit', 0))
        password = get_option(args, 'password', 'rarpassword')
        attempt_decrypt = strtobool(get_option(args, 'attemptdecrypt', 'rarattemptdecrypt', 'false'))
        temp_dir = get_option(args, 'tempdir', 'tempdir', '/tmp/laikaboss_tmp')
        if not os.path.isdir(temp_dir):
            os.mkdir(temp_dir)
            os.chmod(temp_dir, 0777)

        # A temp file must be created as UnRAR2 does not accept buffers
        with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:
            temp_file.write(scanObject.buffer)
            temp_file.flush()

            # RAR can be password protected, which encrypts the headers
            headers_are_encrypted = False
            # RAR can encrypt the files while leaving the headers decrypted
            files_are_encrypted = False

            rar = None
            # list of the file info objects
            infos = []

            try:
                logging.debug('%s: Attempting to open rar file', self.module_name)
                # If headers are encrypted, the following will raise IncorrectRARPassword
                rar = UnRAR2.RarFile(temp_file.name)
                infos = rar.infolist()
                logging.debug('%s: Succeeded opening rar file', self.module_name)

                # If files are encrypted, the filename will be prefixed with a '*'
                for info in infos:
                    if info.filename.startswith('*'):
                        logging.debug('%s: Rar files are encrypted', self.module_name)
                        scanObject.addFlag('ENCRYPTED_RAR')
                        scanObject.addMetadata(self.module_name, "Encrypted", "Protected Files")
                        files_are_encrypted = True
                        break
            except IncorrectRARPassword:
                logging.debug('%s: Rar headers are encrypted', self.module_name)
                scanObject.addFlag('ENCRYPTED_RAR')
                scanObject.addMetadata(self.module_name, "Encrypted", "Protected Header")
                headers_are_encrypted = True
            except InvalidRARArchive:
                logging.debug('%s: Invalid Rar file')

            if (headers_are_encrypted or files_are_encrypted) and attempt_decrypt:
                logging.debug('%s: Attempting to decrypt', self.module_name)
                possible_passwords = []

                # Passwords are sometimes sent in the email content. Use the content of the parent
                # object as the list of possible passwords
                parent_object = getParentObject(result, scanObject)
                if parent_object:
                    possible_passwords = _create_word_list(parent_object.buffer)

                if password:
                    possible_passwords.insert(0, password)

                explode_temp_dir = os.path.join(temp_dir, 'exploderar')
                for possible_password in possible_passwords:
                    try:
                        logging.debug("EXPLODE_RAR: Attempting password '%s'", possible_password)
                        rar = UnRAR2.RarFile(temp_file.name, password=possible_password)
                        # Extraction is needed to force the exception on encrypted files
                        if files_are_encrypted:
                            rar.extract(path=explode_temp_dir)
                        infos = rar.infolist()
                        logging.debug("EXPLODE_RAR: Found password '%s'", possible_password)
                        scanObject.addFlag('rar:decrypted')
                        scanObject.addMetadata(self.module_name, 'Password', possible_password)
                        break
                    except IncorrectRARPassword:
                        continue
                if os.path.exists(explode_temp_dir):
                    remove_dir(explode_temp_dir)

            scanObject.addMetadata(self.module_name, "Total_Files", len(infos))
            file_count = 0
            exceeded_byte_limit = False
            for info in infos:
                if byte_limit and info.size > byte_limit:
                    logging.debug("EXPLODE_RAR: skipping file due to byte limit")
                    exceeded_byte_limit = True
                    continue
                try:
                    content = rar.read_files(info.filename)[0][1]
                    if byte_limit and len(content) > byte_limit:
                        logging.debug("EXPLODE_RAR: skipping file due to byte limit")
                        exceeded_byte_limit = True
                        continue
                    moduleResult.append(ModuleObject(buffer=content,
                        externalVars=ExternalVars(filename=info.filename)))
                except IndexError:
                    pass
                file_count += 1
                if file_limit and file_count >= file_limit:
                    scanObject.addFlag("rar:err:LIMIT_EXCEEDED")
                    logging.debug("EXPLODE_RAR: breaking due to file limit")
                    break
            if exceeded_byte_limit:
                scanObject.addFlag("rar:err:BYTE_LIMIT_EXCEEDED")

        scanObject.addMetadata(self.module_name, "Unzipped", len(moduleResult))
        return moduleResult
Beispiel #14
0
    def run(self, scanObject, result, depth, args):
        """Wrapper method around _run for error handling"""
        moduleLoggingBool = config.modulelogging
        try:
            starttime = time.time()
            if moduleLoggingBool:
                log_module("START", self.module_name, 0, scanObject, result) 

            # Get a configured timeout
            timeout_seconds = int(get_option(args, 'module_timeout', 'global_module_timeout', 3600))

            with timeout(timeout_seconds, exception=GlobalModuleTimeoutError):
                moduleResult = self._run(scanObject, result, depth, args)

            if moduleLoggingBool:
                log_module("END", self.module_name, time.time() - starttime, scanObject, result) 
            if type(moduleResult) is not list:
                msg = "%s returned an object with type %s. Only lists are allowed! Skipping this result." % (self.module_name, type(moduleResult))
                logging.debug(msg)
                if moduleLoggingBool:
                    log_module_error(self.module_name,
                                     scanObject,
                                     result,
                                     msg)
                return []

            return moduleResult 

        except GlobalScanTimeoutError:
            raise
        except GlobalModuleTimeoutError:
            # If the _module times out, add a flag and continue as a normal error
            scanObject.addFlag("dispatch:err:module_timeout:%s" % self.module_name)

            exc_type, exc_value, exc_traceback = sys.exc_info()
            logging.exception("error on %s running _module %s. exception details below: ",
                get_scanObjectUID(getRootObject(result)), self.module_name)

            if moduleLoggingBool:
                log_module_error(self.module_name,
                                 scanObject,
                                 result,
                                 repr(traceback.format_exception(exc_type, 
                                                                 exc_value, 
                                                                 exc_traceback)))
            return []

        except QuitScanException:
            # If the _module is terminated early, add a flag and proceed the exception up the stack
            scanObject.addFlag("dispatch:err:quit_scan")
            logging.warning("quitting scan while running _module %s", self.module_name)
            raise

        except Exception:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            logging.exception("error on %s running _module %s. exception details below: ",
                get_scanObjectUID(getRootObject(result)), self.module_name)

            if moduleLoggingBool:
                log_module_error(self.module_name,
                                 scanObject,
                                 result,
                                 repr(traceback.format_exception(exc_type, 
                                                                 exc_value, 
                                                                 exc_traceback)))
            return [] 
Beispiel #15
0
    def _run(self, scanObject, result, depth, args):
        '''
        Here is where the actual magic happens. This method is called on the object being scanned given that the dispatcher has a matching rule triggering this module.

        There are four types of actions that modules do on scan objects:
          * Add flags.
          * Add metadata.
          * Explode children.
          * Interact with external systems.

        Any module can perform as many of these actions as necessary, and many do more than one, such as adding flags and metadata.

        The parameters provided to this method by the framework are:
          * scanObject: This is the object currently being scanned (duh) of type ScanObject. It contains all of the flags, metadata and other assigned fields regarding this spefic instance of the object. This parameter is used by modules to access the buffer, add flags and add metadata.
          * result: This is the umbrella object that contains all of the ScanObjects created during the root object's scan. It is primarily used by modules to access the parent object of the scanObject when needed.
          * depth: This is a leftover parameter from the tracking of the depth of recursion of objects. It is recommended to get this value from the scanObject itself.
          * args: This is a dictionary of the runtime arguments provided to the module by the dispatcher. This parameter provides customization of module runs from the dispatcher, so that the module can operate differently based on the type/context of the scanObject.

        This method MUST return a list of ModuleObjects. ModuleObjects represent the children exploded (or the less violent extracted) from the scanObject by this module. If the module does not explode any children (as most do not), simply return an empty list. Not returning a list causes the framework to log an error for this module each time it is run, but will not prevent it from running next time, nor will it remove any flags/metadata added by the module run.
        '''
        # This variable is recommended to be used by all modules as the returned list of ModuleObjects, populated as the children objects are found.
        moduleResult = []

        # A typical first step is define the configuration options of the module.
        # A best practice for configuration options to a module is to honor them in this precedence:
        #   3. value set as default in this code
        #   2. value specified in config file
        #   1. value specified in arguments of the invocation of this module (via the dispatcher)
        # To help with this, the get_option(...) method provided in the laikaboss.util module provides a single method call to set the option according to this precedence.
        helloworld_param = int(get_option(args, 'param', 'helloworldparam',
                                          10))

        # To add flags to the object, use the addFlag method of the scanObject.
        # Flags should have the following three parts, separated by ':'s:
        #   * Shortened name of the module.
        #   * 'nfo' if the flag is informational, 'err' if the flag is for a policy/logic error (versus a programatic error), or leave blank if a typical flag.
        #   * Expressive name representing the atomic concept of the flag.
        scanObject.addFlag('e_helloworld:nfo:justsayinghello')

        # To add metadata to the object, use the addMetadata method of the scanObject.
        scanObject.addMetadata(self.module_name, "minsize", helloworld_param)

        # If you want to call a separate function, pass the data and let the
        # function set flags on the data. The function will also modify the moduleResult variable
        # to add subobjects
        flags = self._helloworld(scanObject.buffer, moduleResult,
                                 helloworld_param)

        for flag in flags:
            scanObject.addFlag(flag)

        # Whenever you need to do a try-except, you must make sure to catch and raise the framework ScanError exceptions.
        try:
            nonexistant_var.bad_method_call()
        except NameError:
            pass
        except ScanError:
            raise

        # Always return a list of ModuleObjects (or empty list if no children)
        return moduleResult
Beispiel #16
0
    def run(self, scanObject, result, depth, args):
        '''Wrapper method around _run for error handling'''
        moduleLoggingBool = config.modulelogging
        try:
            starttime = time.time()
            if moduleLoggingBool:
                log_module("START", self.module_name, 0, scanObject, result) 

            # Get a configured timeout
            timeout_seconds = int(get_option(args, 'module_timeout', 'global_module_timeout', 3600))

            with timeout(timeout_seconds, exception=GlobalModuleTimeoutError):
                moduleResult = self._run(scanObject, result, depth, args)

            if moduleLoggingBool:
                log_module("END", self.module_name, time.time() - starttime, scanObject, result) 
            if type(moduleResult) is not list:
                msg = "%s returned an object with type %s. Only lists are allowed! Skipping this result." % (self.module_name, type(moduleResult))
                logging.debug(msg)
                if moduleLoggingBool:
                    log_module_error(self.module_name,
                                     scanObject,
                                     result,
                                     msg)
                return []

            return moduleResult 

        except GlobalScanTimeoutError:
            raise
        except GlobalModuleTimeoutError:
            # If the module times out, add a flag and continue as a normal error
            scanObject.addFlag("dispatch:err:module_timeout:%s" % self.module_name)

            exc_type, exc_value, exc_traceback = sys.exc_info()
            logging.exception("error on %s running module %s. exception details below: ", 
                get_scanObjectUID(getRootObject(result)), self.module_name)

            if moduleLoggingBool:
                log_module_error(self.module_name,
                                 scanObject,
                                 result,
                                 repr(traceback.format_exception(exc_type, 
                                                                 exc_value, 
                                                                 exc_traceback)))
            return []

        except QuitScanException:
            # If the module is terminated early, add a flag and proceed the exception up the stack
            scanObject.addFlag("dispatch:err:quit_scan")
            logging.warn("quitting scan while running module %s", self.module_name)
            raise

        except Exception:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            logging.exception("error on %s running module %s. exception details below: ",
                get_scanObjectUID(getRootObject(result)), self.module_name)

            if moduleLoggingBool:
                log_module_error(self.module_name,
                                 scanObject,
                                 result,
                                 repr(traceback.format_exception(exc_type, 
                                                                 exc_value, 
                                                                 exc_traceback)))
            return []