def _process_module_queue(moduleQueue, result, depth, scanObject, earlyQuitTime=0):
"""
Description: Takes a priority module queue and runs each module in the appropriate order.
Each module is tracked for uniqueness to prevent redundancy.
"""
MAXDEPTH = 0
if hasattr(config, "maxdepth"):
# If the depth limit has been exceeded, then don't run any modules
MAXDEPTH = int(config.maxdepth)
if MAXDEPTH < 0:
MAXDEPTH = 0
moduleSeen = []
while True:
if MAXDEPTH and depth > MAXDEPTH:
errorText = "Depth has been exceeded. Only the dispatcher will be run on this object."
logging.debug(errorText)
log_module_error("si_dispatch", scanObject, result, errorText)
scanObject.addFlag("dispatch:nfo:max_depth_exceeded")
break
# Read until the queue is empty
if moduleQueue.empty():
logging.debug("Module run queue is empty")
break
scanModules = moduleQueue.get()[1]
for sm in scanModules:
if sm in moduleSeen:
logging.debug("Already ran %s, continuing to the next module" % (sm))
continue
module, args = get_module_arguments(sm)
_run_module(module, scanObject, result, depth, args, earlyQuitTime)
moduleSeen.append(sm)
def _process_module_queue(moduleQueue,
result,
depth,
scanObject,
earlyQuitTime=0):
'''
Description: Takes a priority module queue and runs each module in the appropriate order.
Each module is tracked for uniqueness to prevent redundancy.
'''
MAXDEPTH = 0
if hasattr(config, 'maxdepth'):
# If the depth limit has been exceeded, then don't run any modules
MAXDEPTH = int(config.maxdepth)
if MAXDEPTH < 0:
MAXDEPTH = 0
moduleSeen = []
while True:
if MAXDEPTH and depth > MAXDEPTH:
errorText = "Depth has been exceeded. Only the dispatcher will be run on this object."
logging.debug(errorText)
log_module_error("si_dispatch", scanObject, result, errorText)
scanObject.addFlag("dispatch:nfo:max_depth_exceeded")
break
# Read until the queue is empty
if moduleQueue.empty():
logging.debug("Module run queue is empty")
break
scanModules = moduleQueue.get()[1]
for sm in scanModules:
if sm in moduleSeen:
logging.debug("Already ran %s, continuing to the next module" %
(sm))
continue
module, args = get_module_arguments(sm)
_run_module(module, scanObject, result, depth, args, earlyQuitTime)
moduleSeen.append(sm)
def Dispatch(buffer,
result,
depth,
externalVars=None,
scanObject=None,
extScanModules=None,
conditional=False):
"""
Description: By default, this function uses yara to disposition a buffer and determine what scan modules
should be run against it. The function may be called recursively if a scan module returns
additional buffers to scan. The function collects all results into the original result object
passed in by the caller for easy retrieval.
Arguments: (* denotes OPTIONAL parameters):
- buffer: the binary contents of the current object
- result: collects scan results of all objects being scanned in a dictionary
- depth: every time dispatch is called, depth is increased by 1. may be used to limit recursion.
- *externalVars: variables passed in from the caller or other modules
- *scanModules: this function may be called with predefined scan modules set (string, space delimited)
- *conditional: determines whether this function has been called as the result of a conditional scan
- *externalVars: these variables are passed to yara along with the current object to aid in disposition
"""
skip_timeout = True
if depth == 0 or (externalVars is not None
and int(externalVars.depth) > 0):
skip_timeout = False
global_scan_timeout = 3600
if hasattr(config, 'global_scan_timeout'):
global_scan_timeout = int(config.global_scan_timeout)
global_scan_timeout_endtime = result.startTime + global_scan_timeout
if externalVars is not None and externalVars.depth:
depth = externalVars.depth
starttime = time.time()
MAXBYTES = 0
if hasattr(config, 'dispatchmaxbytes'):
# If the depth limit has been exceeded, then don't run any modules
MAXBYTES = int(config.dispatchmaxbytes)
if MAXBYTES < 0:
MAXBYTES = 0
logging.debug('setting dispatch byte limit to %i' % (MAXBYTES))
#
# This branch is designed for first-pass scanning where file type and scan modules are unknown
# Yara is used to disposition the file and determine which modules should be run against it
# Using the result of each module, it is determined (using a separate yara scan on the flags)
# whether or not a conditional scan needs to be run.
if extScanModules is None:
# Generate the scan object from the parameters
scanObject = _gather_metadata(buffer, externalVars, result, depth,
MAXBYTES)
# Increase the depth only if it is the first time scanning an object
depth += 1
logging.debug(
"si_dispatch - Attempting to dispatch - uid: %s, filename: %s, \
source module: %s" % (get_scanObjectUID(scanObject), externalVars.filename,
externalVars.sourceModule))
# check to see if this object has a parent, get the modules run against the parent if it exists
#
externals = {
'ext_parentModules': listToSSV(externalVars.parentModules)
or 'NONE',
'ext_sourceModule': externalVars.sourceModule or 'NONE',
'ext_contentType': listToSSV(scanObject.contentType) or 'NONE',
'ext_filename': externalVars.filename or 'NONE',
'ext_timestamp': externalVars.timestamp or 'NONE',
'ext_source': externalVars.source or 'NONE',
'ext_flags': listToSSV(externalVars.flags) or 'NONE',
'ext_size': scanObject.objectSize,
'ext_depth': int(depth) or 0
}
dispatch_rule_start = time.time()
yresults = yara_on_demand(config.yaradispatchrules, buffer, externals,
MAXBYTES)
if config.modulelogging:
log_module("MSG", 'si_dispatch',
time.time() - dispatch_rule_start, scanObject, result,
"")
moduleQueue = _get_module_queue(yresults, result, scanObject, "Rules")
with _with_conditional(skip_timeout) or timeout(
global_scan_timeout, exception=GlobalScanTimeoutError):
try:
_process_module_queue(moduleQueue, result, depth, scanObject,
global_scan_timeout_endtime)
_conditional_scan(scanObject, externalVars, result, depth)
except GlobalScanTimeoutError:
# If the scan times out, add a flag and continue as a normal error
scanObject.addFlag("dispatch:err:scan_timeout")
# If not the root object, raise the exception to halt the parent scan
if depth > 0 and (externalVars is None
or depth > int(externalVars.depth)):
raise
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception("error on %s. exception details below: " % \
(get_scanObjectUID(getRootObject(result))))
log_module_error(
"dispatch:", scanObject, result,
repr(
traceback.format_exception(exc_type, exc_value,
exc_traceback)))
#
# This branch is designed for externally specified scan modules to be run against a buffer.
# It is not necessary to disposition the file type with yara because we are trusting the caller.
# This branch differs from a conditional scan in that there is no metadata about the buffer already,
# so it must be gathered before beginning the scan. It is also subject to conditional scanning.
elif extScanModules is not None and not conditional:
scanObject = _gather_metadata(buffer, externalVars, result, depth,
MAXBYTES)
with _with_conditional(skip_timeout) or timeout(
global_scan_timeout, exception=GlobalScanTimeoutError):
try:
for sm in extScanModules:
module, args = get_module_arguments(sm)
_run_module(module, scanObject, result, depth, args)
# Disable conditional scan
#_conditional_scan(scanObject, externalVars, result, depth)
except GlobalScanTimeoutError:
# If the scan times out, add a flag and continue as a normal error
scanObject.addFlag("dispatch:err:scan_timeout")
# If not the root object, raise the exception to halt the parent scan
if depth > 0 and (externalVars is None
or depth > int(externalVars.depth)):
raise
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception("error on %s. exception details below: " % \
(get_scanObjectUID(getRootObject(result))))
log_module_error(
"dispatch", scanObject, result,
repr(
traceback.format_exception(exc_type, exc_value,
exc_traceback)))
#
# This branch is specifically for conditional scans kicked off by this function. Metadata about
# the object has already been collected and all that needs to occur is scans by the specified modules.
else:
_process_module_queue(extScanModules, result, depth, scanObject)
logging.debug("si_dispatch - depth: %s, time: %s" %
(depth, time.time() - starttime))
def Dispatch(buffer, result, depth, externalVars=None, scanObject=None, extScanModules=None, conditional=False):
"""
Description: By default, this function uses yara to disposition a buffer and determine what scan modules
should be run against it. The function may be called recursively if a scan module returns
additional buffers to scan. The function collects all results into the original result object
passed in by the caller for easy retrieval.
Arguments: (* denotes OPTIONAL parameters):
- buffer: the binary contents of the current object
- result: collects scan results of all objects being scanned in a dictionary
- depth: every time dispatch is called, depth is increased by 1. may be used to limit recursion.
- *externalVars: variables passed in from the caller or other modules
- *scanModules: this function may be called with predefined scan modules set (string, space delimited)
- *conditional: determines whether this function has been called as the result of a conditional scan
- *externalVars: these variables are passed to yara along with the current object to aid in disposition
"""
skip_timeout = True
if depth == 0 or (externalVars is not None and int(externalVars.depth) > 0):
skip_timeout = False
global_scan_timeout = 3600
if hasattr(config, "global_scan_timeout"):
global_scan_timeout = int(config.global_scan_timeout)
global_scan_timeout_endtime = result.startTime + global_scan_timeout
if externalVars is not None and externalVars.depth:
depth = externalVars.depth
starttime = time.time()
MAXBYTES = 0
if hasattr(config, "dispatchmaxbytes"):
# If the depth limit has been exceeded, then don't run any modules
MAXBYTES = int(config.dispatchmaxbytes)
if MAXBYTES < 0:
MAXBYTES = 0
logging.debug("setting dispatch byte limit to %i" % (MAXBYTES))
#
# This branch is designed for first-pass scanning where file type and scan modules are unknown
# Yara is used to disposition the file and determine which modules should be run against it
# Using the result of each module, it is determined (using a separate yara scan on the flags)
# whether or not a conditional scan needs to be run.
if extScanModules is None:
# Generate the scan object from the parameters
scanObject = _gather_metadata(buffer, externalVars, result, depth, MAXBYTES)
# Increase the depth only if it is the first time scanning an object
depth += 1
logging.debug(
"si_dispatch - Attempting to dispatch - uid: %s, filename: %s, \
source module: %s"
% (get_scanObjectUID(scanObject), externalVars.filename, externalVars.sourceModule)
)
# check to see if this object has a parent, get the modules run against the parent if it exists
#
externals = {
"ext_parentModules": listToSSV(externalVars.parentModules) or "NONE",
"ext_sourceModule": externalVars.sourceModule or "NONE",
"ext_contentType": listToSSV(scanObject.contentType) or "NONE",
"ext_filename": externalVars.filename or "NONE",
"ext_timestamp": externalVars.timestamp or "NONE",
"ext_source": externalVars.source or "NONE",
"ext_flags": listToSSV(externalVars.flags) or "NONE",
"ext_size": scanObject.objectSize,
"ext_depth": int(depth) or 0,
}
dispatch_rule_start = time.time()
yresults = yara_on_demand(config.yaradispatchrules, buffer, externals, MAXBYTES)
if config.modulelogging:
log_module("MSG", "si_dispatch", time.time() - dispatch_rule_start, scanObject, result, "")
moduleQueue = _get_module_queue(yresults, result, scanObject, "Rules")
with _with_conditional(skip_timeout) or timeout(global_scan_timeout, exception=GlobalScanTimeoutError):
try:
_process_module_queue(moduleQueue, result, depth, scanObject, global_scan_timeout_endtime)
_conditional_scan(scanObject, externalVars, result, depth)
except GlobalScanTimeoutError:
# If the scan times out, add a flag and continue as a normal error
scanObject.addFlag("dispatch:err:scan_timeout")
# If not the root object, raise the exception to halt the parent scan
if depth > 0 and (externalVars is None or depth > int(externalVars.depth)):
raise
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception("error on %s. exception details below: " % (get_scanObjectUID(getRootObject(result))))
log_module_error(
"dispatch:",
scanObject,
result,
repr(traceback.format_exception(exc_type, exc_value, exc_traceback)),
)
#
# This branch is designed for externally specified scan modules to be run against a buffer.
# It is not necessary to disposition the file type with yara because we are trusting the caller.
# This branch differs from a conditional scan in that there is no metadata about the buffer already,
# so it must be gathered before beginning the scan. It is also subject to conditional scanning.
elif extScanModules is not None and not conditional:
scanObject = _gather_metadata(buffer, externalVars, result, depth, MAXBYTES)
with _with_conditional(skip_timeout) or timeout(global_scan_timeout, exception=GlobalScanTimeoutError):
try:
for sm in extScanModules:
module, args = get_module_arguments(sm)
_run_module(module, scanObject, result, depth, args)
# Disable conditional scan
# _conditional_scan(scanObject, externalVars, result, depth)
except GlobalScanTimeoutError:
# If the scan times out, add a flag and continue as a normal error
scanObject.addFlag("dispatch:err:scan_timeout")
# If not the root object, raise the exception to halt the parent scan
if depth > 0 and (externalVars is None or depth > int(externalVars.depth)):
raise
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception("error on %s. exception details below: " % (get_scanObjectUID(getRootObject(result))))
log_module_error(
"dispatch", scanObject, result, repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
)
#
# This branch is specifically for conditional scans kicked off by this function. Metadata about
# the object has already been collected and all that needs to occur is scans by the specified modules.
else:
_process_module_queue(extScanModules, result, depth, scanObject)
logging.debug("si_dispatch - depth: %s, time: %s" % (depth, time.time() - starttime))