Example #1
0
def _conditional_scan(scanObject, externalVars, result, depth):
    '''
    Description: This function performs a second pass scan of an object based on the results of a 
                 previous scan only. The yara rules look at the flags applied to this object and 
                 determine any additional scanning that may need to be performed based on these 
                 flags.
    Arguments:
     - scanObject: the current object being scanned
     - result: collects scan results of all objects being scanned in a dictionary
     - depth: every time dispatch is called, depth is increased by 1. may be used to limit recursion.
    '''
    # Attempt to disposition based on flags from the first scan
    try:
        logging.debug(
            "attempting conditional disposition on %s with %s uID: %s parent: %s"
            % (scanObject.filename, listToSSV(scanObject.flags),
               get_scanObjectUID(scanObject), scanObject.parent))
        externals = {
            'ext_parentModules': listToSSV(externalVars.parentModules)
            or 'NONE',
            'ext_sourceModule': externalVars.sourceModule or 'NONE',
            'ext_contentType': listToSSV(scanObject.contentType) or 'NONE',
            'ext_fileType': listToSSV(scanObject.fileType) or 'NONE',
            'ext_filename': externalVars.filename or 'NONE',
            'ext_timestamp': externalVars.timestamp or 'NONE',
            'ext_source': externalVars.source or 'NONE',
            'ext_size': scanObject.objectSize,
            'ext_depth': depth or 0
        }
        yresults = yara_on_demand(config.yaraconditionalrules,
                                  listToSSV(scanObject.flags), externals)
        moduleQueue = _get_module_queue(yresults, result, scanObject,
                                        "Conditional Rules")
    except (QuitScanException, GlobalScanTimeoutError):
        raise
    except Exception:
        logging.exception(
            "si_dispatch: ERROR occured on conditional_scan on UID:%s Check your configuration!",
            get_scanObjectUID(scanObject))
        log_module_error(
            "si_dispatch", scanObject, result,
            "error during conditional_scan: %s" % traceback.format_exc())
        return
    # Recusively call the Dispatcher if any conditional scans need to be performed.
    if not moduleQueue.empty():
        Dispatch(scanObject.buffer,
                 result,
                 depth,
                 scanObject=scanObject,
                 extScanModules=moduleQueue,
                 conditional=True)
Example #2
0
def _conditional_scan(scanObject, externalVars, result, depth):
    """
    Description: This function performs a second pass scan of an object based on the results of a 
                 previous scan only. The yara rules look at the flags applied to this object and 
                 determine any additional scanning that may need to be performed based on these 
                 flags.
    Arguments:
     - scanObject: the current object being scanned
     - result: collects scan results of all objects being scanned in a dictionary
     - depth: every time dispatch is called, depth is increased by 1. may be used to limit recursion.
    """
    # Attempt to disposition based on flags from the first scan
    try:
        logging.debug(
            "attempting conditional disposition on %s with %s uID: %s parent: %s"
            % (scanObject.filename, listToSSV(scanObject.flags), get_scanObjectUID(scanObject), scanObject.parent)
        )
        externals = {
            "ext_parentModules": listToSSV(externalVars.parentModules) or "NONE",
            "ext_sourceModule": externalVars.sourceModule or "NONE",
            "ext_contentType": listToSSV(scanObject.contentType) or "NONE",
            "ext_fileType": listToSSV(scanObject.fileType) or "NONE",
            "ext_filename": externalVars.filename or "NONE",
            "ext_timestamp": externalVars.timestamp or "NONE",
            "ext_source": externalVars.source or "NONE",
            "ext_size": scanObject.objectSize,
            "ext_depth": depth or 0,
        }
        yresults = yara_on_demand(config.yaraconditionalrules, listToSSV(scanObject.flags), externals)
        moduleQueue = _get_module_queue(yresults, result, scanObject, "Conditional Rules")
    except (QuitScanException, GlobalScanTimeoutError):
        raise
    except Exception:
        logging.exception(
            "si_dispatch: ERROR occured on conditional_scan on UID:%s Check your configuration!",
            get_scanObjectUID(scanObject),
        )
        log_module_error(
            "si_dispatch", scanObject, result, "error during conditional_scan: %s" % traceback.format_exc()
        )
        return
    # Recusively call the Dispatcher if any conditional scans need to be performed.
    if not moduleQueue.empty():
        Dispatch(scanObject.buffer, result, depth, scanObject=scanObject, extScanModules=moduleQueue, conditional=True)
Example #3
0
def Dispatch(buffer,
             result,
             depth,
             externalVars=None,
             scanObject=None,
             extScanModules=None,
             conditional=False):
    """
    Description: By default, this function uses yara to disposition a buffer and determine what scan modules
                 should be run against it. The function may be called recursively if a scan module returns 
                 additional buffers to scan. The function collects all results into the original result object
                 passed in by the caller for easy retrieval.
    Arguments: (* denotes OPTIONAL parameters):
     - buffer: the binary contents of the current object
     - result: collects scan results of all objects being scanned in a dictionary
     - depth: every time dispatch is called, depth is increased by 1. may be used to limit recursion.
     - *externalVars: variables passed in from the caller or other modules
     - *scanModules: this function may be called with predefined scan modules set (string, space delimited)
     - *conditional: determines whether this function has been called as the result of a conditional scan
     - *externalVars: these variables are passed to yara along with the current object to aid in disposition
    """

    skip_timeout = True
    if depth == 0 or (externalVars is not None
                      and int(externalVars.depth) > 0):
        skip_timeout = False

    global_scan_timeout = 3600
    if hasattr(config, 'global_scan_timeout'):
        global_scan_timeout = int(config.global_scan_timeout)
    global_scan_timeout_endtime = result.startTime + global_scan_timeout

    if externalVars is not None and externalVars.depth:
        depth = externalVars.depth

    starttime = time.time()
    MAXBYTES = 0
    if hasattr(config, 'dispatchmaxbytes'):
        # If the depth limit has been exceeded, then don't run any modules
        MAXBYTES = int(config.dispatchmaxbytes)
        if MAXBYTES < 0:
            MAXBYTES = 0
        logging.debug('setting dispatch byte limit to %i' % (MAXBYTES))

    #
    #  This branch is designed for first-pass scanning where file type and scan modules are unknown
    #  Yara is used to disposition the file and determine which modules should be run against it
    #  Using the result of each module, it is determined (using a separate yara scan on the flags)
    #  whether or not a conditional scan needs to be run.
    if extScanModules is None:

        # Generate the scan object from the parameters
        scanObject = _gather_metadata(buffer, externalVars, result, depth,
                                      MAXBYTES)

        # Increase the depth only if it is the first time scanning an object
        depth += 1

        logging.debug(
            "si_dispatch - Attempting to dispatch - uid: %s, filename: %s, \
source module: %s" % (get_scanObjectUID(scanObject), externalVars.filename,
                      externalVars.sourceModule))
        #  check to see if this object has a parent, get the modules run against the parent if it exists
        #
        externals = {
            'ext_parentModules': listToSSV(externalVars.parentModules)
            or 'NONE',
            'ext_sourceModule': externalVars.sourceModule or 'NONE',
            'ext_contentType': listToSSV(scanObject.contentType) or 'NONE',
            'ext_filename': externalVars.filename or 'NONE',
            'ext_timestamp': externalVars.timestamp or 'NONE',
            'ext_source': externalVars.source or 'NONE',
            'ext_flags': listToSSV(externalVars.flags) or 'NONE',
            'ext_size': scanObject.objectSize,
            'ext_depth': int(depth) or 0
        }

        dispatch_rule_start = time.time()
        yresults = yara_on_demand(config.yaradispatchrules, buffer, externals,
                                  MAXBYTES)
        if config.modulelogging:
            log_module("MSG", 'si_dispatch',
                       time.time() - dispatch_rule_start, scanObject, result,
                       "")
        moduleQueue = _get_module_queue(yresults, result, scanObject, "Rules")

        with _with_conditional(skip_timeout) or timeout(
                global_scan_timeout, exception=GlobalScanTimeoutError):
            try:
                _process_module_queue(moduleQueue, result, depth, scanObject,
                                      global_scan_timeout_endtime)
                _conditional_scan(scanObject, externalVars, result, depth)
            except GlobalScanTimeoutError:
                # If the scan times out, add a flag and continue as a normal error
                scanObject.addFlag("dispatch:err:scan_timeout")

                # If not the root object, raise the exception to halt the parent scan
                if depth > 0 and (externalVars is None
                                  or depth > int(externalVars.depth)):
                    raise

                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.exception("error on %s. exception details below: " % \
                    (get_scanObjectUID(getRootObject(result))))

                log_module_error(
                    "dispatch:", scanObject, result,
                    repr(
                        traceback.format_exception(exc_type, exc_value,
                                                   exc_traceback)))

    #
    #  This branch is designed for externally specified scan modules to be run against a buffer.
    #  It is not necessary to disposition the file type with yara because we are trusting the caller.
    #  This branch differs from a conditional scan in that there is no metadata about the buffer already,
    #  so it must be gathered before beginning the scan. It is also subject to conditional scanning.
    elif extScanModules is not None and not conditional:
        scanObject = _gather_metadata(buffer, externalVars, result, depth,
                                      MAXBYTES)
        with _with_conditional(skip_timeout) or timeout(
                global_scan_timeout, exception=GlobalScanTimeoutError):
            try:
                for sm in extScanModules:
                    module, args = get_module_arguments(sm)
                    _run_module(module, scanObject, result, depth, args)
                    # Disable conditional scan
                    #_conditional_scan(scanObject, externalVars, result, depth)
            except GlobalScanTimeoutError:
                # If the scan times out, add a flag and continue as a normal error
                scanObject.addFlag("dispatch:err:scan_timeout")

                # If not the root object, raise the exception to halt the parent scan
                if depth > 0 and (externalVars is None
                                  or depth > int(externalVars.depth)):
                    raise

                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.exception("error on %s. exception details below: " % \
                    (get_scanObjectUID(getRootObject(result))))

                log_module_error(
                    "dispatch", scanObject, result,
                    repr(
                        traceback.format_exception(exc_type, exc_value,
                                                   exc_traceback)))

    #
    #  This branch is specifically for conditional scans kicked off by this function. Metadata about
    #  the object has already been collected and all that needs to occur is scans by the specified modules.
    else:
        _process_module_queue(extScanModules, result, depth, scanObject)

    logging.debug("si_dispatch - depth: %s, time: %s" %
                  (depth, time.time() - starttime))
Example #4
0
def Dispatch(buffer, result, depth, externalVars=None, scanObject=None, extScanModules=None, conditional=False):
    """
    Description: By default, this function uses yara to disposition a buffer and determine what scan modules
                 should be run against it. The function may be called recursively if a scan module returns 
                 additional buffers to scan. The function collects all results into the original result object
                 passed in by the caller for easy retrieval.
    Arguments: (* denotes OPTIONAL parameters):
     - buffer: the binary contents of the current object
     - result: collects scan results of all objects being scanned in a dictionary
     - depth: every time dispatch is called, depth is increased by 1. may be used to limit recursion.
     - *externalVars: variables passed in from the caller or other modules
     - *scanModules: this function may be called with predefined scan modules set (string, space delimited)
     - *conditional: determines whether this function has been called as the result of a conditional scan
     - *externalVars: these variables are passed to yara along with the current object to aid in disposition
    """

    skip_timeout = True
    if depth == 0 or (externalVars is not None and int(externalVars.depth) > 0):
        skip_timeout = False

    global_scan_timeout = 3600
    if hasattr(config, "global_scan_timeout"):
        global_scan_timeout = int(config.global_scan_timeout)
    global_scan_timeout_endtime = result.startTime + global_scan_timeout

    if externalVars is not None and externalVars.depth:
        depth = externalVars.depth

    starttime = time.time()
    MAXBYTES = 0
    if hasattr(config, "dispatchmaxbytes"):
        # If the depth limit has been exceeded, then don't run any modules
        MAXBYTES = int(config.dispatchmaxbytes)
        if MAXBYTES < 0:
            MAXBYTES = 0
        logging.debug("setting dispatch byte limit to %i" % (MAXBYTES))

    #
    #  This branch is designed for first-pass scanning where file type and scan modules are unknown
    #  Yara is used to disposition the file and determine which modules should be run against it
    #  Using the result of each module, it is determined (using a separate yara scan on the flags)
    #  whether or not a conditional scan needs to be run.
    if extScanModules is None:

        # Generate the scan object from the parameters
        scanObject = _gather_metadata(buffer, externalVars, result, depth, MAXBYTES)

        # Increase the depth only if it is the first time scanning an object
        depth += 1

        logging.debug(
            "si_dispatch - Attempting to dispatch - uid: %s, filename: %s, \
source module: %s"
            % (get_scanObjectUID(scanObject), externalVars.filename, externalVars.sourceModule)
        )
        #  check to see if this object has a parent, get the modules run against the parent if it exists
        #
        externals = {
            "ext_parentModules": listToSSV(externalVars.parentModules) or "NONE",
            "ext_sourceModule": externalVars.sourceModule or "NONE",
            "ext_contentType": listToSSV(scanObject.contentType) or "NONE",
            "ext_filename": externalVars.filename or "NONE",
            "ext_timestamp": externalVars.timestamp or "NONE",
            "ext_source": externalVars.source or "NONE",
            "ext_flags": listToSSV(externalVars.flags) or "NONE",
            "ext_size": scanObject.objectSize,
            "ext_depth": int(depth) or 0,
        }

        dispatch_rule_start = time.time()
        yresults = yara_on_demand(config.yaradispatchrules, buffer, externals, MAXBYTES)
        if config.modulelogging:
            log_module("MSG", "si_dispatch", time.time() - dispatch_rule_start, scanObject, result, "")
        moduleQueue = _get_module_queue(yresults, result, scanObject, "Rules")

        with _with_conditional(skip_timeout) or timeout(global_scan_timeout, exception=GlobalScanTimeoutError):
            try:
                _process_module_queue(moduleQueue, result, depth, scanObject, global_scan_timeout_endtime)
                _conditional_scan(scanObject, externalVars, result, depth)
            except GlobalScanTimeoutError:
                # If the scan times out, add a flag and continue as a normal error
                scanObject.addFlag("dispatch:err:scan_timeout")

                # If not the root object, raise the exception to halt the parent scan
                if depth > 0 and (externalVars is None or depth > int(externalVars.depth)):
                    raise

                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.exception("error on %s. exception details below: " % (get_scanObjectUID(getRootObject(result))))

                log_module_error(
                    "dispatch:",
                    scanObject,
                    result,
                    repr(traceback.format_exception(exc_type, exc_value, exc_traceback)),
                )

    #
    #  This branch is designed for externally specified scan modules to be run against a buffer.
    #  It is not necessary to disposition the file type with yara because we are trusting the caller.
    #  This branch differs from a conditional scan in that there is no metadata about the buffer already,
    #  so it must be gathered before beginning the scan. It is also subject to conditional scanning.
    elif extScanModules is not None and not conditional:
        scanObject = _gather_metadata(buffer, externalVars, result, depth, MAXBYTES)
        with _with_conditional(skip_timeout) or timeout(global_scan_timeout, exception=GlobalScanTimeoutError):
            try:
                for sm in extScanModules:
                    module, args = get_module_arguments(sm)
                    _run_module(module, scanObject, result, depth, args)
                    # Disable conditional scan
                    # _conditional_scan(scanObject, externalVars, result, depth)
            except GlobalScanTimeoutError:
                # If the scan times out, add a flag and continue as a normal error
                scanObject.addFlag("dispatch:err:scan_timeout")

                # If not the root object, raise the exception to halt the parent scan
                if depth > 0 and (externalVars is None or depth > int(externalVars.depth)):
                    raise

                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.exception("error on %s. exception details below: " % (get_scanObjectUID(getRootObject(result))))

                log_module_error(
                    "dispatch", scanObject, result, repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
                )

    #
    #  This branch is specifically for conditional scans kicked off by this function. Metadata about
    #  the object has already been collected and all that needs to occur is scans by the specified modules.
    else:
        _process_module_queue(extScanModules, result, depth, scanObject)

    logging.debug("si_dispatch - depth: %s, time: %s" % (depth, time.time() - starttime))