def _rewrite_config(ModuleList, config, filepath=CONFIG):
"""
Write in default config for all modules.
ModuleList - The list of modules
config - The config object
"""
filepath = determine_configuration_path(filepath)
if VERBOSE:
print('Rewriting config...')
ModuleList.sort()
for module in ModuleList:
if module.endswith('.py'):
modname = os.path.basename(module).split('.')[0]
moddir = os.path.dirname(module)
mod = load_module(os.path.basename(module).split('.')[0], [moddir])
if mod:
try:
conf = mod.DEFAULTCONF
except Exception as e:
# TODO: log exception
continue
_update_DEFAULTCONF(conf, filepath)
config.add_section(modname)
for key in conf:
config.set(modname, key, str(conf[key]))
_update_DEFAULTCONF(DEFAULTCONF, filepath)
config.add_section('main')
for key in DEFAULTCONF:
config.set('main', key, str(DEFAULTCONF[key]))
with codecs.open(filepath, 'w', 'utf-8') as f:
config.write(f)
def setup(self):
m = utils.load_module('test_1', [multiscanner.MODULESDIR])
global_module_interface = multiscanner._GlobalModuleInterface()
self.result = multiscanner._run_module('test_1', m, self.filelist,
self.threadDict,
global_module_interface)
global_module_interface._cleanup()
def _rewrite_config(ModuleList, config, filepath=CONFIG):
"""
Write in default config for all modules.
ModuleList - The list of modules
config - The config object
"""
filepath = determine_configuration_path(filepath)
if VERBOSE:
print('Rewriting config...')
ModuleList.sort()
for module in ModuleList:
if module.endswith('.py'):
modname = os.path.basename(module).split('.')[0]
moddir = os.path.dirname(module)
mod = load_module(os.path.basename(module).split('.')[0], [moddir])
if mod:
try:
conf = mod.DEFAULTCONF
except Exception as e:
# TODO: log exception
continue
_update_DEFAULTCONF(conf, filepath)
config.add_section(modname)
for key in conf:
config.set(modname, key, str(conf[key]))
_update_DEFAULTCONF(DEFAULTCONF, filepath)
config.add_section('main')
for key in DEFAULTCONF:
config.set('main', key, str(DEFAULTCONF[key]))
with codecs.open(filepath, 'w', 'utf-8') as f:
config.write(f)
def _write_missing_module_configs(ModuleList, config, filepath=CONFIG):
"""
Write in default config for modules not in config file. Returns True if config was written, False if not.
ModuleList - The list of modules
config - The config object
"""
filepath = determine_configuration_path(filepath)
ConfNeedsWrite = False
ModuleList.sort()
for module in ModuleList:
if module.endswith(".py"):
modname = os.path.basename(module).split('.')[0]
moddir = os.path.dirname(module)
if modname not in config.sections():
mod = load_module(
os.path.basename(module).split('.')[0], [moddir])
if mod:
try:
conf = mod.DEFAULTCONF
except Exception as e:
# TODO: log exception
continue
ConfNeedsWrite = True
_update_DEFAULTCONF(conf, filepath)
config.add_section(modname)
for key in conf:
config.set(modname, key, str(conf[key]))
if 'main' not in config.sections():
ConfNeedsWrite = True
_update_DEFAULTCONF(DEFAULTCONF, filepath)
config.add_section('main')
for key in DEFAULTCONF:
config.set('main', key, str(DEFAULTCONF[key]))
if ConfNeedsWrite:
with codecs.open(filepath, 'w', 'utf-8') as f:
config.write(f)
return True
return False
def _get_storage_classes(dir_path=STORAGE_DIR):
storage_classes = {}
dir_list = utils.parseDir(dir_path, recursive=True)
dir_list.remove(os.path.join(dir_path, 'storage.py'))
# dir_list.remove(os.path.join(dir_path, '__init__.py'))
dir_list.remove(os.path.join(dir_path, 'sql_driver.py'))
for filename in dir_list:
if filename.endswith('.py'):
modname = os.path.basename(filename[:-3])
moddir = os.path.dirname(filename)
mod = utils.load_module(os.path.basename(modname), [moddir])
if not mod:
print(filename, 'not a valid storage module...')
continue
for member_name in dir(mod):
member = getattr(mod, member_name)
if inspect.isclass(member) and issubclass(member, Storage):
storage_classes[member_name] = member()
if 'Storage' in storage_classes:
del storage_classes['Storage']
return storage_classes
def _write_missing_module_configs(ModuleList, config, filepath=CONFIG):
"""
Write in default config for modules not in config file. Returns True if config was written, False if not.
ModuleList - The list of modules
config - The config object
"""
filepath = determine_configuration_path(filepath)
ConfNeedsWrite = False
ModuleList.sort()
for module in ModuleList:
if module.endswith(".py"):
modname = os.path.basename(module).split('.')[0]
moddir = os.path.dirname(module)
if modname not in config.sections():
mod = load_module(os.path.basename(module).split('.')[0], [moddir])
if mod:
try:
conf = mod.DEFAULTCONF
except Exception as e:
# TODO: log exception
continue
ConfNeedsWrite = True
_update_DEFAULTCONF(conf, filepath)
config.add_section(modname)
for key in conf:
config.set(modname, key, str(conf[key]))
if 'main' not in config.sections():
ConfNeedsWrite = True
_update_DEFAULTCONF(DEFAULTCONF, filepath)
config.add_section('main')
for key in DEFAULTCONF:
config.set('main', key, str(DEFAULTCONF[key]))
if ConfNeedsWrite:
with codecs.open(filepath, 'w', 'utf-8') as f:
config.write(f)
return True
return False
def _start_module_threads(filelist, ModuleList, config, global_module_interface):
"""
Starts each module on the file list in a separate thread. Returns a list of threads
filelist - A lists of strings. The strings are files to be scanned
ModuleList - A list of all the modules to be run
config - The config dictionary
global_module_interface - The global module interface to be injected in each module
"""
if VERBOSE:
print("Starting modules...")
ThreadList = []
ThreadDict = {}
global_module_interface.run_count += 1
# Starts a thread for each module.
for module in ModuleList:
if module.endswith(".py"):
modname = os.path.basename(module[:-3])
# If the module is disabled we don't mess with it further to prevent spamming errors on screen
if modname in config:
if not config[modname].get('ENABLED', True):
continue
moddir = os.path.dirname(module)
mod = load_module(os.path.basename(module).split('.')[0], [moddir])
if not mod:
print(module, " not a valid module...")
continue
conf = None
if modname in config:
if '_load_default' in config or '_load_default' in config[modname]:
try:
conf = mod.DEFAULTCONF
conf.update(config[modname])
except Exception as e:
# TODO: log exception
conf = config[modname]
# Remove _load_default from config
if '_load_default' in conf:
del conf['_load_default']
else:
conf = config[modname]
# Try and read in the default conf if one was not passed
if not conf:
try:
conf = mod.DEFAULTCONF
except Exception as e:
# TODO: log exception
pass
thread = _Thread(
target=_run_module,
args=(modname, mod, filelist, ThreadDict, global_module_interface, conf))
thread.name = modname
thread.setDaemon(True)
ThreadList.append(thread)
ThreadDict[modname] = thread
for thread in ThreadList:
thread.start()
return ThreadList
def setup(self):
self.m = utils.load_module('test_2', [multiscanner.MODULESDIR])
self.threadDict['test_2'] = mock.Mock()
self.threadDict['test_1'] = mock.Mock()
self.threadDict['test_1'].ret = ([('a', 'a'), ('C:\\c', 'c')], {})
self.global_module_interface = multiscanner._GlobalModuleInterface()
def setup(self):
m = utils.load_module('test_1', [multiscanner.MODULESDIR])
global_module_interface = multiscanner._GlobalModuleInterface()
self.result = multiscanner._run_module('test_1', m, self.filelist, self.threadDict, global_module_interface)
global_module_interface._cleanup()
def test_fail_loadModule():
"""Ensure _loadModule works"""
m = utils.load_module('notathing', [os.path.join(CWD, "modules")])
assert m is None
def test_loadModule():
"""Ensure _loadModule works"""
m = utils.load_module('test_1', [os.path.join(CWD, "modules")])
assert isinstance(m, types.ModuleType)
def setup(self):
self.m = utils.load_module('test_2', [multiscanner.MODULESDIR])
self.threadDict['test_2'] = mock.Mock()
self.threadDict['test_1'] = mock.Mock()
self.threadDict['test_1'].ret = ([('a', 'a'), ('C:\\c', 'c')], {})
self.global_module_interface = multiscanner._GlobalModuleInterface()
def test_fail_loadModule():
"""Ensure _loadModule works"""
m = utils.load_module('notathing', [os.path.join(CWD, "modules")])
assert m is None
def test_loadModule():
"""Ensure _loadModule works"""
m = utils.load_module('test_1', [os.path.join(CWD, "modules")])
assert isinstance(m, types.ModuleType)
def _start_module_threads(filelist, ModuleList, config,
global_module_interface):
"""
Starts each module on the file list in a separate thread. Returns a list of threads
filelist - A lists of strings. The strings are files to be scanned
ModuleList - A list of all the modules to be run
config - The config dictionary
global_module_interface - The global module interface to be injected in each module
"""
if VERBOSE:
print("Starting modules...")
ThreadList = []
ThreadDict = {}
global_module_interface.run_count += 1
# Starts a thread for each module.
for module in ModuleList:
if module.endswith(".py"):
modname = os.path.basename(module[:-3])
# If the module is disabled we don't mess with it further to prevent spamming errors on screen
if modname in config:
if not config[modname].get('ENABLED', True):
continue
moddir = os.path.dirname(module)
mod = load_module(os.path.basename(module).split('.')[0], [moddir])
if not mod:
print(module, " not a valid module...")
continue
conf = None
if modname in config:
if '_load_default' in config or '_load_default' in config[
modname]:
try:
conf = mod.DEFAULTCONF
conf.update(config[modname])
except Exception as e:
# TODO: log exception
conf = config[modname]
# Remove _load_default from config
if '_load_default' in conf:
del conf['_load_default']
else:
conf = config[modname]
# Try and read in the default conf if one was not passed
if not conf:
try:
conf = mod.DEFAULTCONF
except Exception as e:
# TODO: log exception
pass
thread = _Thread(target=_run_module,
args=(modname, mod, filelist, ThreadDict,
global_module_interface, conf))
thread.name = modname
thread.setDaemon(True)
ThreadList.append(thread)
ThreadDict[modname] = thread
for thread in ThreadList:
thread.start()
return ThreadList
