Esempio n. 1
0
def _process_module_queue(moduleQueue, result, depth, scanObject, earlyQuitTime=0):
    """
    Description: Takes a priority module queue and runs each module in the appropriate order.
                 Each module is tracked for uniqueness to prevent redundancy.
    """

    MAXDEPTH = 0
    if hasattr(config, "maxdepth"):
        # If the depth limit has been exceeded, then don't run any modules
        MAXDEPTH = int(config.maxdepth)
        if MAXDEPTH < 0:
            MAXDEPTH = 0

    moduleSeen = []
    while True:
        if MAXDEPTH and depth > MAXDEPTH:
            errorText = "Depth has been exceeded. Only the dispatcher will be run on this object."
            logging.debug(errorText)
            log_module_error("si_dispatch", scanObject, result, errorText)
            scanObject.addFlag("dispatch:nfo:max_depth_exceeded")
            break

        # Read until the queue is empty
        if moduleQueue.empty():
            logging.debug("Module run queue is empty")
            break
        scanModules = moduleQueue.get()[1]
        for sm in scanModules:
            if sm in moduleSeen:
                logging.debug("Already ran %s, continuing to the next module" % (sm))
                continue
            module, args = get_module_arguments(sm)
            _run_module(module, scanObject, result, depth, args, earlyQuitTime)
            moduleSeen.append(sm)
Esempio n. 2
0
def _process_module_queue(moduleQueue,
                          result,
                          depth,
                          scanObject,
                          earlyQuitTime=0):
    '''
    Description: Takes a priority module queue and runs each module in the appropriate order.
                 Each module is tracked for uniqueness to prevent redundancy.
    '''

    MAXDEPTH = 0
    if hasattr(config, 'maxdepth'):
        # If the depth limit has been exceeded, then don't run any modules
        MAXDEPTH = int(config.maxdepth)
        if MAXDEPTH < 0:
            MAXDEPTH = 0

    moduleSeen = []
    while True:
        if MAXDEPTH and depth > MAXDEPTH:
            errorText = "Depth has been exceeded. Only the dispatcher will be run on this object."
            logging.debug(errorText)
            log_module_error("si_dispatch", scanObject, result, errorText)
            scanObject.addFlag("dispatch:nfo:max_depth_exceeded")
            break

        # Read until the queue is empty
        if moduleQueue.empty():
            logging.debug("Module run queue is empty")
            break
        scanModules = moduleQueue.get()[1]
        for sm in scanModules:
            if sm in moduleSeen:
                logging.debug("Already ran %s, continuing to the next module" %
                              (sm))
                continue
            module, args = get_module_arguments(sm)
            _run_module(module, scanObject, result, depth, args, earlyQuitTime)
            moduleSeen.append(sm)
Esempio n. 3
0
def Dispatch(buffer,
             result,
             depth,
             externalVars=None,
             scanObject=None,
             extScanModules=None,
             conditional=False):
    """
    Description: By default, this function uses yara to disposition a buffer and determine what scan modules
                 should be run against it. The function may be called recursively if a scan module returns 
                 additional buffers to scan. The function collects all results into the original result object
                 passed in by the caller for easy retrieval.
    Arguments: (* denotes OPTIONAL parameters):
     - buffer: the binary contents of the current object
     - result: collects scan results of all objects being scanned in a dictionary
     - depth: every time dispatch is called, depth is increased by 1. may be used to limit recursion.
     - *externalVars: variables passed in from the caller or other modules
     - *scanModules: this function may be called with predefined scan modules set (string, space delimited)
     - *conditional: determines whether this function has been called as the result of a conditional scan
     - *externalVars: these variables are passed to yara along with the current object to aid in disposition
    """

    skip_timeout = True
    if depth == 0 or (externalVars is not None
                      and int(externalVars.depth) > 0):
        skip_timeout = False

    global_scan_timeout = 3600
    if hasattr(config, 'global_scan_timeout'):
        global_scan_timeout = int(config.global_scan_timeout)
    global_scan_timeout_endtime = result.startTime + global_scan_timeout

    if externalVars is not None and externalVars.depth:
        depth = externalVars.depth

    starttime = time.time()
    MAXBYTES = 0
    if hasattr(config, 'dispatchmaxbytes'):
        # If the depth limit has been exceeded, then don't run any modules
        MAXBYTES = int(config.dispatchmaxbytes)
        if MAXBYTES < 0:
            MAXBYTES = 0
        logging.debug('setting dispatch byte limit to %i' % (MAXBYTES))

    #
    #  This branch is designed for first-pass scanning where file type and scan modules are unknown
    #  Yara is used to disposition the file and determine which modules should be run against it
    #  Using the result of each module, it is determined (using a separate yara scan on the flags)
    #  whether or not a conditional scan needs to be run.
    if extScanModules is None:

        # Generate the scan object from the parameters
        scanObject = _gather_metadata(buffer, externalVars, result, depth,
                                      MAXBYTES)

        # Increase the depth only if it is the first time scanning an object
        depth += 1

        logging.debug(
            "si_dispatch - Attempting to dispatch - uid: %s, filename: %s, \
source module: %s" % (get_scanObjectUID(scanObject), externalVars.filename,
                      externalVars.sourceModule))
        #  check to see if this object has a parent, get the modules run against the parent if it exists
        #
        externals = {
            'ext_parentModules': listToSSV(externalVars.parentModules)
            or 'NONE',
            'ext_sourceModule': externalVars.sourceModule or 'NONE',
            'ext_contentType': listToSSV(scanObject.contentType) or 'NONE',
            'ext_filename': externalVars.filename or 'NONE',
            'ext_timestamp': externalVars.timestamp or 'NONE',
            'ext_source': externalVars.source or 'NONE',
            'ext_flags': listToSSV(externalVars.flags) or 'NONE',
            'ext_size': scanObject.objectSize,
            'ext_depth': int(depth) or 0
        }

        dispatch_rule_start = time.time()
        yresults = yara_on_demand(config.yaradispatchrules, buffer, externals,
                                  MAXBYTES)
        if config.modulelogging:
            log_module("MSG", 'si_dispatch',
                       time.time() - dispatch_rule_start, scanObject, result,
                       "")
        moduleQueue = _get_module_queue(yresults, result, scanObject, "Rules")

        with _with_conditional(skip_timeout) or timeout(
                global_scan_timeout, exception=GlobalScanTimeoutError):
            try:
                _process_module_queue(moduleQueue, result, depth, scanObject,
                                      global_scan_timeout_endtime)
                _conditional_scan(scanObject, externalVars, result, depth)
            except GlobalScanTimeoutError:
                # If the scan times out, add a flag and continue as a normal error
                scanObject.addFlag("dispatch:err:scan_timeout")

                # If not the root object, raise the exception to halt the parent scan
                if depth > 0 and (externalVars is None
                                  or depth > int(externalVars.depth)):
                    raise

                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.exception("error on %s. exception details below: " % \
                    (get_scanObjectUID(getRootObject(result))))

                log_module_error(
                    "dispatch:", scanObject, result,
                    repr(
                        traceback.format_exception(exc_type, exc_value,
                                                   exc_traceback)))

    #
    #  This branch is designed for externally specified scan modules to be run against a buffer.
    #  It is not necessary to disposition the file type with yara because we are trusting the caller.
    #  This branch differs from a conditional scan in that there is no metadata about the buffer already,
    #  so it must be gathered before beginning the scan. It is also subject to conditional scanning.
    elif extScanModules is not None and not conditional:
        scanObject = _gather_metadata(buffer, externalVars, result, depth,
                                      MAXBYTES)
        with _with_conditional(skip_timeout) or timeout(
                global_scan_timeout, exception=GlobalScanTimeoutError):
            try:
                for sm in extScanModules:
                    module, args = get_module_arguments(sm)
                    _run_module(module, scanObject, result, depth, args)
                    # Disable conditional scan
                    #_conditional_scan(scanObject, externalVars, result, depth)
            except GlobalScanTimeoutError:
                # If the scan times out, add a flag and continue as a normal error
                scanObject.addFlag("dispatch:err:scan_timeout")

                # If not the root object, raise the exception to halt the parent scan
                if depth > 0 and (externalVars is None
                                  or depth > int(externalVars.depth)):
                    raise

                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.exception("error on %s. exception details below: " % \
                    (get_scanObjectUID(getRootObject(result))))

                log_module_error(
                    "dispatch", scanObject, result,
                    repr(
                        traceback.format_exception(exc_type, exc_value,
                                                   exc_traceback)))

    #
    #  This branch is specifically for conditional scans kicked off by this function. Metadata about
    #  the object has already been collected and all that needs to occur is scans by the specified modules.
    else:
        _process_module_queue(extScanModules, result, depth, scanObject)

    logging.debug("si_dispatch - depth: %s, time: %s" %
                  (depth, time.time() - starttime))
Esempio n. 4
0
def Dispatch(buffer, result, depth, externalVars=None, scanObject=None, extScanModules=None, conditional=False):
    """
    Description: By default, this function uses yara to disposition a buffer and determine what scan modules
                 should be run against it. The function may be called recursively if a scan module returns 
                 additional buffers to scan. The function collects all results into the original result object
                 passed in by the caller for easy retrieval.
    Arguments: (* denotes OPTIONAL parameters):
     - buffer: the binary contents of the current object
     - result: collects scan results of all objects being scanned in a dictionary
     - depth: every time dispatch is called, depth is increased by 1. may be used to limit recursion.
     - *externalVars: variables passed in from the caller or other modules
     - *scanModules: this function may be called with predefined scan modules set (string, space delimited)
     - *conditional: determines whether this function has been called as the result of a conditional scan
     - *externalVars: these variables are passed to yara along with the current object to aid in disposition
    """

    skip_timeout = True
    if depth == 0 or (externalVars is not None and int(externalVars.depth) > 0):
        skip_timeout = False

    global_scan_timeout = 3600
    if hasattr(config, "global_scan_timeout"):
        global_scan_timeout = int(config.global_scan_timeout)
    global_scan_timeout_endtime = result.startTime + global_scan_timeout

    if externalVars is not None and externalVars.depth:
        depth = externalVars.depth

    starttime = time.time()
    MAXBYTES = 0
    if hasattr(config, "dispatchmaxbytes"):
        # If the depth limit has been exceeded, then don't run any modules
        MAXBYTES = int(config.dispatchmaxbytes)
        if MAXBYTES < 0:
            MAXBYTES = 0
        logging.debug("setting dispatch byte limit to %i" % (MAXBYTES))

    #
    #  This branch is designed for first-pass scanning where file type and scan modules are unknown
    #  Yara is used to disposition the file and determine which modules should be run against it
    #  Using the result of each module, it is determined (using a separate yara scan on the flags)
    #  whether or not a conditional scan needs to be run.
    if extScanModules is None:

        # Generate the scan object from the parameters
        scanObject = _gather_metadata(buffer, externalVars, result, depth, MAXBYTES)

        # Increase the depth only if it is the first time scanning an object
        depth += 1

        logging.debug(
            "si_dispatch - Attempting to dispatch - uid: %s, filename: %s, \
source module: %s"
            % (get_scanObjectUID(scanObject), externalVars.filename, externalVars.sourceModule)
        )
        #  check to see if this object has a parent, get the modules run against the parent if it exists
        #
        externals = {
            "ext_parentModules": listToSSV(externalVars.parentModules) or "NONE",
            "ext_sourceModule": externalVars.sourceModule or "NONE",
            "ext_contentType": listToSSV(scanObject.contentType) or "NONE",
            "ext_filename": externalVars.filename or "NONE",
            "ext_timestamp": externalVars.timestamp or "NONE",
            "ext_source": externalVars.source or "NONE",
            "ext_flags": listToSSV(externalVars.flags) or "NONE",
            "ext_size": scanObject.objectSize,
            "ext_depth": int(depth) or 0,
        }

        dispatch_rule_start = time.time()
        yresults = yara_on_demand(config.yaradispatchrules, buffer, externals, MAXBYTES)
        if config.modulelogging:
            log_module("MSG", "si_dispatch", time.time() - dispatch_rule_start, scanObject, result, "")
        moduleQueue = _get_module_queue(yresults, result, scanObject, "Rules")

        with _with_conditional(skip_timeout) or timeout(global_scan_timeout, exception=GlobalScanTimeoutError):
            try:
                _process_module_queue(moduleQueue, result, depth, scanObject, global_scan_timeout_endtime)
                _conditional_scan(scanObject, externalVars, result, depth)
            except GlobalScanTimeoutError:
                # If the scan times out, add a flag and continue as a normal error
                scanObject.addFlag("dispatch:err:scan_timeout")

                # If not the root object, raise the exception to halt the parent scan
                if depth > 0 and (externalVars is None or depth > int(externalVars.depth)):
                    raise

                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.exception("error on %s. exception details below: " % (get_scanObjectUID(getRootObject(result))))

                log_module_error(
                    "dispatch:",
                    scanObject,
                    result,
                    repr(traceback.format_exception(exc_type, exc_value, exc_traceback)),
                )

    #
    #  This branch is designed for externally specified scan modules to be run against a buffer.
    #  It is not necessary to disposition the file type with yara because we are trusting the caller.
    #  This branch differs from a conditional scan in that there is no metadata about the buffer already,
    #  so it must be gathered before beginning the scan. It is also subject to conditional scanning.
    elif extScanModules is not None and not conditional:
        scanObject = _gather_metadata(buffer, externalVars, result, depth, MAXBYTES)
        with _with_conditional(skip_timeout) or timeout(global_scan_timeout, exception=GlobalScanTimeoutError):
            try:
                for sm in extScanModules:
                    module, args = get_module_arguments(sm)
                    _run_module(module, scanObject, result, depth, args)
                    # Disable conditional scan
                    # _conditional_scan(scanObject, externalVars, result, depth)
            except GlobalScanTimeoutError:
                # If the scan times out, add a flag and continue as a normal error
                scanObject.addFlag("dispatch:err:scan_timeout")

                # If not the root object, raise the exception to halt the parent scan
                if depth > 0 and (externalVars is None or depth > int(externalVars.depth)):
                    raise

                exc_type, exc_value, exc_traceback = sys.exc_info()
                logging.exception("error on %s. exception details below: " % (get_scanObjectUID(getRootObject(result))))

                log_module_error(
                    "dispatch", scanObject, result, repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
                )

    #
    #  This branch is specifically for conditional scans kicked off by this function. Metadata about
    #  the object has already been collected and all that needs to occur is scans by the specified modules.
    else:
        _process_module_queue(extScanModules, result, depth, scanObject)

    logging.debug("si_dispatch - depth: %s, time: %s" % (depth, time.time() - starttime))