def _run(self, scanObject, result, depth, args):
'''Laika framework module logic execution'''
moduleResult = []
# Determine file limit from arguments
file_limit = int(get_option(args, 'filelimit', 'zipfilelimit', 0))
byte_limit = int(get_option(args, 'bytelimit', 'zipbytelimit', 0))
zippw = get_option(args, 'password', 'zippassword', '')
# write temporary file so we can open it with zipfile
file = cStringIO.StringIO()
file.write(scanObject.buffer)
try:
logging.debug("first attempt at unzipping..")
self._unzip_file(self, moduleResult, file, scanObject, result, zippw, file_limit, byte_limit)
except zipfile.BadZipfile:
try:
# try to repair the zip file (known python limitation)
logging.debug("error extracting zip, trying to fix it")
self._fix_bad_zip(file, scanObject.buffer)
self._unzip_file(self, moduleResult, file, scanObject, result, zippw, file_limit, byte_limit)
#except QuitScanException:
# raise
except:
# add a flag to the object to indicate it couldn't be extracted
logging.debug("couldn't fix zip, marking it as corrupt")
scanObject.addFlag("CORRUPT_ZIP")
# error logging handled by SI_MODULE wrapper
raise
finally:
scanObject.addMetadata(self.module_name, "Unzipped", len(moduleResult))
file.close()
return moduleResult
def _run(self, scanObject, result, depth, args):
"""Laika framework _module logic execution"""
moduleResult = []
# Determine file limit from arguments
file_limit = int(get_option(args, 'filelimit', 'zipfilelimit', 0))
byte_limit = int(get_option(args, 'bytelimit', 'zipbytelimit', 0))
zippw = get_option(args, 'password', 'zippassword', '')
# write temporary file so we can open it with zipfile
file = cStringIO.StringIO()
file.write(scanObject.buffer)
try:
logging.debug("first attempt at unzipping..")
self._unzip_file(self, moduleResult, file, scanObject, result, zippw, file_limit, byte_limit)
except zipfile.BadZipfile:
try:
# try to repair the zip file (known python limitation)
logging.debug("error extracting zip, trying to fix it")
self._fix_bad_zip(file, scanObject.buffer)
self._unzip_file(self, moduleResult, file, scanObject, result, zippw, file_limit, byte_limit)
#except QuitScanException:
# raise
except:
# add a flag to the object to indicate it couldn't be extracted
logging.debug("couldn't fix zip, marking it as corrupt")
scanObject.addFlag("CORRUPT_ZIP")
# error logging handled by SI_MODULE wrapper
raise
finally:
scanObject.addMetadata(self.module_name, "Unzipped", len(moduleResult))
file.close()
return moduleResult
def _run(self, scanObject, result, depth, args):
'''
Arguments:
unix_socket -- Path to the clamd unix socket (str)
max_bytes -- Maximum number of bytes to scan (0 is unlimited) (int)
timeout -- Max number of seconds to scan (float)
Returns:
Flags -- Virus name hits
Metadata -- Unix socket or daemon errors
'''
moduleResult = []
# Defualt max of 20 MB
default_max_bytes = 20000000
# Default timeout of 10.0 seconds
default_timeout = 10.0
# Default clamd unix socket
default_unix_socket = '/var/run/clamav/clamd.sock'
unix_socket = str(get_option(args, 'unixsocket', 'scanclamavunixsocket', default_unix_socket))
max_bytes = int(get_option(args, 'maxbytes', 'scanclamavmaxbytes', default_max_bytes))
timeout = float(get_option(args, 'timeout', 'scanclamavtimeout', default_timeout))
if timeout < 0.01:
timeout = default_timeout
# Connect to daemon
try:
clam = pyclamd.ClamdUnixSocket(filename=unix_socket, timeout=timeout)
except IOError:
logging.debug('IOError: Cannot connect to clamd unix socket file')
scanObject.addMetadata(self.module_name, 'Error', 'IOError: clamd socket')
return moduleResult
errmsg = None
try:
# Scan the buffer with clamav
if max_bytes <= 0:
clam_result = clam.scan_stream(scanObject.buffer)
else:
clam_result = clam.scan_stream(str(buffer(scanObject.buffer, 0, max_bytes)))
# Process a result
if clam_result:
status, virusname = clam_result['stream']
scanObject.addFlag("%s:%s" % (self.flag_prefix, str(virusname)))
except ValueError as e:
errmsg = "ValueError (BufferTooLong): %s" % e
except IOError as e:
errmsg = "IOError (ScanError): %s" % e
except Exception as e:
errmsg = "Unexpected error: %s" % e
if errmsg:
logging.debug(errmsg)
scanObject.addMetadata(self.module_name, 'Error', errmsg)
return moduleResult
def _run(self, scanObject, result, depth, args):
'''
Assumes:
there is a string like object in scanObject.buffer
Ensures:
hash values added using scanObject.addMetadata
Laika Config File Options:
hashmd5: "1" = md5.hexdigest, "0" = omit
hashSHA1: "1" = sha1.hexdigest, "0" = omit
hashSHA256: "1" = sha256.hexdigest, "0" = omit
hashSHA512: "1" = sha256.hexdigest, "0" = omit
hashSHA1: "1" = sha1.hexdigest, "0" = omit
ssdeep: "1" = ssdeep.hash, "0" = omit
Function Arguments:
:param scanObject:<laikaboss.objectmodel.ScanObject>
:param result:<laikaboss.objectmodel.ScanResult>
:param depth:<int>
:param args:<dict> --execution flow controls--
Valid args names <str> (value must be 1, 0, "1", or "0")
1/"1": Generate the hash of named type
0/"0": Omit the hash of named type
default args:
{"md5":1,
"SHA1":0,
"SHA256":1,
"SHA512":1,
"ssdeep":0}
:return: Always returns a empty list (no child objects)
'''
moduleResult = []
metaDict = {}
if int(get_option(args, 'md5', 'hashmd5', "md5" in self.module_defaults)):
metaDict['md5'] = hashlib.md5(scanObject.buffer).hexdigest()
if int(get_option(args, 'SHA1', 'hashSHA1', "SHA1" in self.module_defaults)):
metaDict['SHA1'] = hashlib.sha1(scanObject.buffer).hexdigest()
if int(get_option(args, 'SHA256', 'hashSHA256', "SHA256" in self.module_defaults)):
metaDict['SHA256'] = hashlib.sha256(scanObject.buffer).hexdigest()
if int(get_option(args, 'SHA512', 'hashSHA512', "SHA512" in self.module_defaults)):
metaDict['SHA512'] = hashlib.sha512(scanObject.buffer).hexdigest()
if int(get_option(args, 'ssdeep', 'hashssdeep', "ssdeep" in self.module_defaults)):
#only import ssdeep if dispatched.
#Prevents import error if you don't have/want the package
#python should keep handing you the original, minimal/no overhead
try:
import ssdeep
metaDict['ssdeep'] = ssdeep.hash(scanObject.buffer)
except ImportError:
metaDict['ssdeep'] = "" #indicate ssdeep was configured but failed
scanObject.addMetadata(self.module_name, "HASHES", metaDict)
return moduleResult
def _run(self, scanObject, result, depth, args):
"""
Arguments:
unixsocket -- Path to the clamd unix socket (str)
maxbytes -- Maximum number of bytes to scan (0 is unlimited) (int)
Returns:
Flags -- Virus name hits
Metadata -- Unix socket or daemon errors
"""
moduleResult = []
unix_socket = str(
get_option(args, 'unixsocket', 'scanclamavunixsocket',
'/var/run/clamav/clamd.ctl'))
max_bytes = int(
get_option(args, 'maxbytes', 'scanclamavmaxbytes', 20000000))
# Connect to daemon
if not self.clam:
try:
self.clam = pyclamd.ClamdUnixSocket(filename=unix_socket)
except IOError:
logging.debug(
'IOError: Cannot connect to clamd unix socket file')
scanObject.addMetadata(self.module_name, 'Error',
'IOError: clamd socket')
raise
try:
# Scan the buffer with clamav
if max_bytes <= 0:
clam_result = self.clam.scan_stream(scanObject.buffer)
else:
clam_result = self.clam.scan_stream(
str(buffer(scanObject.buffer, 0, max_bytes)))
# Process a result
if clam_result:
status, virusname = clam_result['stream']
scanObject.addFlag("%s:%s" %
(self.flag_prefix, str(virusname)))
except ValueError as e:
scanObject.addMetadata(self.module_name, 'Error',
'ValueError (BufferTooLong): %s' % str(e))
except IOError as e:
scanObject.addMetadata(self.module_name, 'Error',
'IOError (ScanError): %s' % str(e))
return moduleResult
def _run(self, scanObject, result, depth, args):
'''
Here is where the actual magic happens. This method is called on the object being scanned given that the dispatcher has a matching rule triggering this module.
There are four types of actions that modules do on scan objects:
* Add flags.
* Add metadata.
* Explode children.
* Interact with external systems.
Any module can perform as many of these actions as necessary, and many do more than one, such as adding flags and metadata.
The parameters provided to this method by the framework are:
* scanObject: This is the object currently being scanned (duh) of type ScanObject. It contains all of the flags, metadata and other assigned fields regarding this spefic instance of the object. This parameter is used by modules to access the buffer, add flags and add metadata.
* result: This is the umbrella object that contains all of the ScanObjects created during the root object's scan. It is primarily used by modules to access the parent object of the scanObject when needed.
* depth: This is a leftover parameter from the tracking of the depth of recursion of objects. It is recommended to get this value from the scanObject itself.
* args: This is a dictionary of the runtime arguments provided to the module by the dispatcher. This parameter provides customization of module runs from the dispatcher, so that the module can operate differently based on the type/context of the scanObject.
This method MUST return a list of ModuleObjects. ModuleObjects represent the children exploded (or the less violent extracted) from the scanObject by this module. If the module does not explode any children (as most do not), simply return an empty list. Not returning a list causes the framework to log an error for this module each time it is run, but will not prevent it from running next time, nor will it remove any flags/metadata added by the module run.
'''
# This variable is recommended to be used by all modules as the returned list of ModuleObjects, populated as the children objects are found.
moduleResult = []
# A typical first step is define the configuration options of the module.
# A best practice for configuration options to a module is to honor them in this precedence:
# 3. value set as default in this code
# 2. value specified in config file
# 1. value specified in arguments of the invocation of this module (via the dispatcher)
# To help with this, the get_option(...) method provided in the laikaboss.util module provides a single method call to set the option according to this precedence.
helloworld_param = int(get_option(args, 'param', 'helloworldparam', 10))
# To add flags to the object, use the addFlag method of the scanObject.
# Flags should have the following three parts, separated by ':'s:
# * Shortened name of the module.
# * 'nfo' if the flag is informational, 'err' if the flag is for a policy/logic error (versus a programatic error), or leave blank if a typical flag.
# * Expressive name representing the atomic concept of the flag.
scanObject.addFlag('e_helloworld:nfo:justsayinghello')
# To add metadata to the object, use the addMetadata method of the scanObject.
scanObject.addMetadata(self.module_name, "minsize", helloworld_param)
# If you want to call a separate function, pass the data and let the
# function set flags on the data. The function will also modify the moduleResult variable
# to add subobjects
flags = self._helloworld(scanObject.buffer, moduleResult, helloworld_param)
for flag in flags:
scanObject.addFlag(flag)
# Whenever you need to do a try-except, you must make sure to catch and raise the framework ScanError exceptions.
try:
nonexistant_var.bad_method_call()
except NameError:
pass
except ScanError:
raise
# Always return a list of ModuleObjects (or empty list if no children)
return moduleResult
def _run(self, scanObject, result, depth, args):
"""
Arguments:
unixsocket -- Path to the clamd unix socket (str)
maxbytes -- Maximum number of bytes to scan (0 is unlimited) (int)
Returns:
Flags -- Virus name hits
Metadata -- Unix socket or daemon errors
"""
moduleResult = []
unix_socket = str(get_option(args, "unixsocket", "scanclamavunixsocket", "/var/run/clamav/clamd.ctl"))
max_bytes = int(get_option(args, "maxbytes", "scanclamavmaxbytes", 20000000))
# Connect to daemon
if not self.clam:
try:
self.clam = pyclamd.ClamdUnixSocket(filename=unix_socket)
except IOError:
logging.debug("IOError: Cannot connect to clamd unix socket file")
scanObject.addMetadata(self.module_name, "Error", "IOError: clamd socket")
raise
try:
# Scan the buffer with clamav
if max_bytes <= 0:
clam_result = self.clam.scan_stream(scanObject.buffer)
else:
clam_result = self.clam.scan_stream(str(buffer(scanObject.buffer, 0, max_bytes)))
# Process a result
if clam_result:
status, virusname = clam_result["stream"]
scanObject.addFlag("%s:%s" % (self.flag_prefix, str(virusname)))
except ValueError as e:
scanObject.addMetadata(self.module_name, "Error", "ValueError (BufferTooLong): %s" % str(e))
except IOError as e:
scanObject.addMetadata(self.module_name, "Error", "IOError (ScanError): %s" % str(e))
return moduleResult
def _run(self, scanObject, result, depth, args):
moduleResult = []
e = email.message_from_string(scanObject.buffer)
attachments = []
i = 1
for p in e.walk():
childBuffer = p.get_payload(decode=True)
if childBuffer is not None:
filename = p.get_filename()
if filename is None:
filename = 'e_email_%s_%s' % (p.get_content_type(), i)
else:
attachments.append(filename)
logging.debug("explode email: found filename: %s" % (filename))
moduleResult.append(
ModuleObject(buffer=childBuffer,
externalVars=ExternalVars(
filename=filename,
contentType=p.get_content_maintype())))
i += 1
# If enabled, this will combine the email headers and all decoded
# text portions contained in the email into a single object
if strtobool(
get_option(args, 'createhybrid', 'explodeemlcreatehybrid',
'False')):
# First, grab the headers
header_end = scanObject.buffer.find('\x0a\x0a')
hybrid = scanObject.buffer[:header_end] + '\n\n'
for mo in moduleResult:
if 'text' in mo.externalVars.contentType:
hybrid += mo.buffer + '\n\n'
# Add the hybrid as another object with a special content type
# for easy identification.
moduleResult.append(
ModuleObject(
buffer=hybrid,
externalVars=ExternalVars(
filename='e_email_hybrid',
contentType='application/x-laika-eml-hybrid')))
# Since we already gathered up the attachment names, we'll add them
# on behalf of META_EMAIL
scanObject.addMetadata('META_EMAIL', 'Attachments', attachments)
return moduleResult
def _run(self, scanObject, result, depth, args):
moduleResult = []
e = email.message_from_string(scanObject.buffer)
attachments = []
i = 1
for p in e.walk():
childBuffer = p.get_payload(decode=True)
if childBuffer is not None:
filename = p.get_filename()
if filename is None:
filename = 'e_email_%s_%s' % (p.get_content_type(),i)
else:
attachments.append(filename)
logging.debug("explode email: found filename: %s" % (filename))
moduleResult.append(ModuleObject(buffer=childBuffer,
externalVars=ExternalVars(filename=filename,
contentType=p.get_content_maintype())))
i += 1
# If enabled, this will combine the email headers and all decoded
# text portions contained in the email into a single object
if strtobool(get_option(args, 'createhybrid', 'explodeemlcreatehybrid', 'False')):
# First, grab the headers
header_end = scanObject.buffer.find('\x0a\x0a')
hybrid = scanObject.buffer[:header_end] + '\n\n'
for mo in moduleResult:
if 'text' in mo.externalVars.contentType:
hybrid += mo.buffer + '\n\n'
# Add the hybrid as another object with a special content type
# for easy identification.
moduleResult.append(ModuleObject(buffer=hybrid,
externalVars=ExternalVars(filename='e_email_hybrid',
contentType='application/x-laika-eml-hybrid')))
# Since we already gathered up the attachment names, we'll add them
# on behalf of META_EMAIL
scanObject.addMetadata('META_EMAIL', 'Attachments', attachments)
return moduleResult
def _run(self, scanObject, result, depth, args):
"""Main module execution. Logs the scan result to fluentd."""
tag = get_option(args, "tag", "fluent_tag", "laikaboss.log_fluent")
host = get_option(args, "host", "fluent_host", "localhost")
port = int(get_option(args, "port", "fluent_port", 24224))
bufmax = int(get_option(args, "bufmax", "fluent_bufmax", 1048576)) # 1 MB
timeout = float(get_option(args, "timeout", "fluent_timeout", 3.0))
label = get_option(args, "label", "fluent_label", "scan_result")
sender = self._get_sender(tag, host, port, bufmax, timeout)
log_record = self._parse_log_record(result)
sender.emit(label, log_record)
if sender.pendings: # buffer in case of failed transmission
log_module_error(self.module_name, scanObject, result,
'Log event failed emition. Will retry on next emition.')
return []
def _run(self, scanObject, result, depth, args):
"""Main _module execution. Logs the scan result to fluentd."""
tag = get_option(args, "tag", "fluent_tag", "laikaboss.log_fluent")
host = get_option(args, "host", "fluent_host", "localhost")
port = int(get_option(args, "port", "fluent_port", 24224))
bufmax = int(get_option(args, "bufmax", "fluent_bufmax",
1048576)) # 1 MB
timeout = float(get_option(args, "timeout", "fluent_timeout", 3.0))
label = get_option(args, "label", "fluent_label", "scan_result")
sender = self._get_sender(tag, host, port, bufmax, timeout)
log_record = self._parse_log_record(result)
sender.emit(label, log_record)
if sender.pendings: # buffer in case of failed transmission
log_module_error(
self.module_name, scanObject, result,
'Log event failed emition. Will retry on next emition.')
return []
def _run(self, scanObject, result, depth, args):
'''Laika framework module logic execution'''
moduleResult = []
file_limit = int(get_option(args, 'filelimit', 'rarfilelimit', 0))
byte_limit = int(get_option(args, 'bytelimit', 'rarbytelimit', 0))
password = get_option(args, 'password', 'rarpassword')
attempt_decrypt = strtobool(
get_option(args, 'attemptdecrypt', 'rarattemptdecrypt', 'false'))
temp_dir = get_option(args, 'tempdir', 'tempdir', '/tmp/laikaboss_tmp')
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
os.chmod(temp_dir, 0777)
# A temp file must be created as UnRAR2 does not accept buffers
with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:
temp_file.write(scanObject.buffer)
temp_file.flush()
# RAR can be password protected, which encrypts the headers
headers_are_encrypted = False
# RAR can encrypt the files while leaving the headers decrypted
files_are_encrypted = False
rar = None
# list of the file info objects
infos = []
try:
logging.debug('%s: Attempting to open rar file',
self.module_name)
# If headers are encrypted, the following will raise IncorrectRARPassword
rar = UnRAR2.RarFile(temp_file.name)
infos = rar.infolist()
logging.debug('%s: Succeeded opening rar file',
self.module_name)
# If files are encrypted, the filename will be prefixed with a '*'
for info in infos:
if info.filename.startswith('*'):
logging.debug('%s: Rar files are encrypted',
self.module_name)
scanObject.addFlag('ENCRYPTED_RAR')
scanObject.addMetadata(self.module_name, "Encrypted",
"Protected Files")
files_are_encrypted = True
break
except IncorrectRARPassword:
logging.debug('%s: Rar headers are encrypted',
self.module_name)
scanObject.addFlag('ENCRYPTED_RAR')
scanObject.addMetadata(self.module_name, "Encrypted",
"Protected Header")
headers_are_encrypted = True
except InvalidRARArchive:
logging.debug('%s: Invalid Rar file')
if (headers_are_encrypted
or files_are_encrypted) and attempt_decrypt:
logging.debug('%s: Attempting to decrypt', self.module_name)
possible_passwords = []
# Passwords are sometimes sent in the email content. Use the content of the parent
# object as the list of possible passwords
parent_object = getParentObject(result, scanObject)
if parent_object:
possible_passwords = _create_word_list(
parent_object.buffer)
if password:
possible_passwords.insert(0, password)
explode_temp_dir = os.path.join(temp_dir, 'exploderar')
for possible_password in possible_passwords:
try:
logging.debug("EXPLODE_RAR: Attempting password '%s'",
possible_password)
rar = UnRAR2.RarFile(temp_file.name,
password=possible_password)
# Extraction is needed to force the exception on encrypted files
if files_are_encrypted:
rar.extract(path=explode_temp_dir)
infos = rar.infolist()
logging.debug("EXPLODE_RAR: Found password '%s'",
possible_password)
scanObject.addFlag('rar:decrypted')
scanObject.addMetadata(self.module_name, 'Password',
possible_password)
break
except IncorrectRARPassword:
continue
if os.path.exists(explode_temp_dir):
remove_dir(explode_temp_dir)
scanObject.addMetadata(self.module_name, "Total_Files", len(infos))
file_count = 0
exceeded_byte_limit = False
for info in infos:
if byte_limit and info.size > byte_limit:
logging.debug(
"EXPLODE_RAR: skipping file due to byte limit")
exceeded_byte_limit = True
continue
try:
content = rar.read_files(info.filename)[0][1]
if byte_limit and len(content) > byte_limit:
logging.debug(
"EXPLODE_RAR: skipping file due to byte limit")
exceeded_byte_limit = True
continue
moduleResult.append(
ModuleObject(
buffer=content,
externalVars=ExternalVars(filename=info.filename)))
except IndexError:
pass
file_count += 1
if file_limit and file_count >= file_limit:
scanObject.addFlag("rar:err:LIMIT_EXCEEDED")
logging.debug("EXPLODE_RAR: breaking due to file limit")
break
if exceeded_byte_limit:
scanObject.addFlag("rar:err:BYTE_LIMIT_EXCEEDED")
scanObject.addMetadata(self.module_name, "Unzipped", len(moduleResult))
return moduleResult
def _run(self, scanObject, result, depth, args):
'''Laika framework module logic execution'''
moduleResult = []
file_limit = int(get_option(args, 'filelimit', 'rarfilelimit', 0))
byte_limit = int(get_option(args, 'bytelimit', 'rarbytelimit', 0))
password = get_option(args, 'password', 'rarpassword')
attempt_decrypt = strtobool(get_option(args, 'attemptdecrypt', 'rarattemptdecrypt', 'false'))
temp_dir = get_option(args, 'tempdir', 'tempdir', '/tmp/laikaboss_tmp')
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
os.chmod(temp_dir, 0777)
# A temp file must be created as UnRAR2 does not accept buffers
with tempfile.NamedTemporaryFile(dir=temp_dir) as temp_file:
temp_file.write(scanObject.buffer)
temp_file.flush()
# RAR can be password protected, which encrypts the headers
headers_are_encrypted = False
# RAR can encrypt the files while leaving the headers decrypted
files_are_encrypted = False
rar = None
# list of the file info objects
infos = []
try:
logging.debug('%s: Attempting to open rar file', self.module_name)
# If headers are encrypted, the following will raise IncorrectRARPassword
rar = UnRAR2.RarFile(temp_file.name)
infos = rar.infolist()
logging.debug('%s: Succeeded opening rar file', self.module_name)
# If files are encrypted, the filename will be prefixed with a '*'
for info in infos:
if info.filename.startswith('*'):
logging.debug('%s: Rar files are encrypted', self.module_name)
scanObject.addFlag('ENCRYPTED_RAR')
scanObject.addMetadata(self.module_name, "Encrypted", "Protected Files")
files_are_encrypted = True
break
except IncorrectRARPassword:
logging.debug('%s: Rar headers are encrypted', self.module_name)
scanObject.addFlag('ENCRYPTED_RAR')
scanObject.addMetadata(self.module_name, "Encrypted", "Protected Header")
headers_are_encrypted = True
except InvalidRARArchive:
logging.debug('%s: Invalid Rar file')
if (headers_are_encrypted or files_are_encrypted) and attempt_decrypt:
logging.debug('%s: Attempting to decrypt', self.module_name)
possible_passwords = []
# Passwords are sometimes sent in the email content. Use the content of the parent
# object as the list of possible passwords
parent_object = getParentObject(result, scanObject)
if parent_object:
possible_passwords = _create_word_list(parent_object.buffer)
if password:
possible_passwords.insert(0, password)
explode_temp_dir = os.path.join(temp_dir, 'exploderar')
for possible_password in possible_passwords:
try:
logging.debug("EXPLODE_RAR: Attempting password '%s'", possible_password)
rar = UnRAR2.RarFile(temp_file.name, password=possible_password)
# Extraction is needed to force the exception on encrypted files
if files_are_encrypted:
rar.extract(path=explode_temp_dir)
infos = rar.infolist()
logging.debug("EXPLODE_RAR: Found password '%s'", possible_password)
scanObject.addFlag('rar:decrypted')
scanObject.addMetadata(self.module_name, 'Password', possible_password)
break
except IncorrectRARPassword:
continue
if os.path.exists(explode_temp_dir):
remove_dir(explode_temp_dir)
scanObject.addMetadata(self.module_name, "Total_Files", len(infos))
file_count = 0
exceeded_byte_limit = False
for info in infos:
if byte_limit and info.size > byte_limit:
logging.debug("EXPLODE_RAR: skipping file due to byte limit")
exceeded_byte_limit = True
continue
try:
content = rar.read_files(info.filename)[0][1]
if byte_limit and len(content) > byte_limit:
logging.debug("EXPLODE_RAR: skipping file due to byte limit")
exceeded_byte_limit = True
continue
moduleResult.append(ModuleObject(buffer=content,
externalVars=ExternalVars(filename=info.filename)))
except IndexError:
pass
file_count += 1
if file_limit and file_count >= file_limit:
scanObject.addFlag("rar:err:LIMIT_EXCEEDED")
logging.debug("EXPLODE_RAR: breaking due to file limit")
break
if exceeded_byte_limit:
scanObject.addFlag("rar:err:BYTE_LIMIT_EXCEEDED")
scanObject.addMetadata(self.module_name, "Unzipped", len(moduleResult))
return moduleResult
def run(self, scanObject, result, depth, args):
"""Wrapper method around _run for error handling"""
moduleLoggingBool = config.modulelogging
try:
starttime = time.time()
if moduleLoggingBool:
log_module("START", self.module_name, 0, scanObject, result)
# Get a configured timeout
timeout_seconds = int(get_option(args, 'module_timeout', 'global_module_timeout', 3600))
with timeout(timeout_seconds, exception=GlobalModuleTimeoutError):
moduleResult = self._run(scanObject, result, depth, args)
if moduleLoggingBool:
log_module("END", self.module_name, time.time() - starttime, scanObject, result)
if type(moduleResult) is not list:
msg = "%s returned an object with type %s. Only lists are allowed! Skipping this result." % (self.module_name, type(moduleResult))
logging.debug(msg)
if moduleLoggingBool:
log_module_error(self.module_name,
scanObject,
result,
msg)
return []
return moduleResult
except GlobalScanTimeoutError:
raise
except GlobalModuleTimeoutError:
# If the _module times out, add a flag and continue as a normal error
scanObject.addFlag("dispatch:err:module_timeout:%s" % self.module_name)
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception("error on %s running _module %s. exception details below: ",
get_scanObjectUID(getRootObject(result)), self.module_name)
if moduleLoggingBool:
log_module_error(self.module_name,
scanObject,
result,
repr(traceback.format_exception(exc_type,
exc_value,
exc_traceback)))
return []
except QuitScanException:
# If the _module is terminated early, add a flag and proceed the exception up the stack
scanObject.addFlag("dispatch:err:quit_scan")
logging.warning("quitting scan while running _module %s", self.module_name)
raise
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception("error on %s running _module %s. exception details below: ",
get_scanObjectUID(getRootObject(result)), self.module_name)
if moduleLoggingBool:
log_module_error(self.module_name,
scanObject,
result,
repr(traceback.format_exception(exc_type,
exc_value,
exc_traceback)))
return []
def _run(self, scanObject, result, depth, args):
'''
Here is where the actual magic happens. This method is called on the object being scanned given that the dispatcher has a matching rule triggering this module.
There are four types of actions that modules do on scan objects:
* Add flags.
* Add metadata.
* Explode children.
* Interact with external systems.
Any module can perform as many of these actions as necessary, and many do more than one, such as adding flags and metadata.
The parameters provided to this method by the framework are:
* scanObject: This is the object currently being scanned (duh) of type ScanObject. It contains all of the flags, metadata and other assigned fields regarding this spefic instance of the object. This parameter is used by modules to access the buffer, add flags and add metadata.
* result: This is the umbrella object that contains all of the ScanObjects created during the root object's scan. It is primarily used by modules to access the parent object of the scanObject when needed.
* depth: This is a leftover parameter from the tracking of the depth of recursion of objects. It is recommended to get this value from the scanObject itself.
* args: This is a dictionary of the runtime arguments provided to the module by the dispatcher. This parameter provides customization of module runs from the dispatcher, so that the module can operate differently based on the type/context of the scanObject.
This method MUST return a list of ModuleObjects. ModuleObjects represent the children exploded (or the less violent extracted) from the scanObject by this module. If the module does not explode any children (as most do not), simply return an empty list. Not returning a list causes the framework to log an error for this module each time it is run, but will not prevent it from running next time, nor will it remove any flags/metadata added by the module run.
'''
# This variable is recommended to be used by all modules as the returned list of ModuleObjects, populated as the children objects are found.
moduleResult = []
# A typical first step is define the configuration options of the module.
# A best practice for configuration options to a module is to honor them in this precedence:
# 3. value set as default in this code
# 2. value specified in config file
# 1. value specified in arguments of the invocation of this module (via the dispatcher)
# To help with this, the get_option(...) method provided in the laikaboss.util module provides a single method call to set the option according to this precedence.
helloworld_param = int(get_option(args, 'param', 'helloworldparam',
10))
# To add flags to the object, use the addFlag method of the scanObject.
# Flags should have the following three parts, separated by ':'s:
# * Shortened name of the module.
# * 'nfo' if the flag is informational, 'err' if the flag is for a policy/logic error (versus a programatic error), or leave blank if a typical flag.
# * Expressive name representing the atomic concept of the flag.
scanObject.addFlag('e_helloworld:nfo:justsayinghello')
# To add metadata to the object, use the addMetadata method of the scanObject.
scanObject.addMetadata(self.module_name, "minsize", helloworld_param)
# If you want to call a separate function, pass the data and let the
# function set flags on the data. The function will also modify the moduleResult variable
# to add subobjects
flags = self._helloworld(scanObject.buffer, moduleResult,
helloworld_param)
for flag in flags:
scanObject.addFlag(flag)
# Whenever you need to do a try-except, you must make sure to catch and raise the framework ScanError exceptions.
try:
nonexistant_var.bad_method_call()
except NameError:
pass
except ScanError:
raise
# Always return a list of ModuleObjects (or empty list if no children)
return moduleResult
def run(self, scanObject, result, depth, args):
'''Wrapper method around _run for error handling'''
moduleLoggingBool = config.modulelogging
try:
starttime = time.time()
if moduleLoggingBool:
log_module("START", self.module_name, 0, scanObject, result)
# Get a configured timeout
timeout_seconds = int(get_option(args, 'module_timeout', 'global_module_timeout', 3600))
with timeout(timeout_seconds, exception=GlobalModuleTimeoutError):
moduleResult = self._run(scanObject, result, depth, args)
if moduleLoggingBool:
log_module("END", self.module_name, time.time() - starttime, scanObject, result)
if type(moduleResult) is not list:
msg = "%s returned an object with type %s. Only lists are allowed! Skipping this result." % (self.module_name, type(moduleResult))
logging.debug(msg)
if moduleLoggingBool:
log_module_error(self.module_name,
scanObject,
result,
msg)
return []
return moduleResult
except GlobalScanTimeoutError:
raise
except GlobalModuleTimeoutError:
# If the module times out, add a flag and continue as a normal error
scanObject.addFlag("dispatch:err:module_timeout:%s" % self.module_name)
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception("error on %s running module %s. exception details below: ",
get_scanObjectUID(getRootObject(result)), self.module_name)
if moduleLoggingBool:
log_module_error(self.module_name,
scanObject,
result,
repr(traceback.format_exception(exc_type,
exc_value,
exc_traceback)))
return []
except QuitScanException:
# If the module is terminated early, add a flag and proceed the exception up the stack
scanObject.addFlag("dispatch:err:quit_scan")
logging.warn("quitting scan while running module %s", self.module_name)
raise
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
logging.exception("error on %s running module %s. exception details below: ",
get_scanObjectUID(getRootObject(result)), self.module_name)
if moduleLoggingBool:
log_module_error(self.module_name,
scanObject,
result,
repr(traceback.format_exception(exc_type,
exc_value,
exc_traceback)))
return []