def __init__(self, CoreObj, Options):
self.Core = CoreObj
#This should be dynamic from filesystem:
#self.PluginGroups = [ 'web', 'net', 'aux' ]
#self.PluginTypes = [ 'passive', 'semi_passive', 'active', 'grep' ]
#self.AllowedPluginTypes = self.GetAllowedPluginTypes(Options['PluginType'].split(','))
#self.Simulation, self.Scope, self.PluginGroup, self.Algorithm, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['Algorithm'], Options['ListPlugins'] ]
self.Simulation, self.Scope, self.PluginGroup, self.ListPlugins = [
Options['Simulation'], Options['Scope'], Options['PluginGroup'],
Options['ListPlugins']
]
self.OnlyPluginsList = self.ValidateAndFormatPluginList(
Options['OnlyPlugins'])
self.ExceptPluginsList = self.ValidateAndFormatPluginList(
Options['ExceptPlugins'])
#print "OnlyPlugins="+str(self.OnlyPluginsList)
#print "ExceptPlugins="+str(self.ExceptPluginsList)
#print "Options['PluginType']="+str(Options['PluginType'])
if isinstance(
Options['PluginType'], str
): # For special plugin types like "quiet" -> "semi_passive" + "passive"
Options['PluginType'] = Options['PluginType'].split(',')
self.AllowedPlugins = self.Core.DB.Plugin.GetPluginsByGroupType(
self.PluginGroup, Options['PluginType'])
self.OnlyPluginsSet = len(self.OnlyPluginsList) > 0
self.ExceptPluginsSet = len(self.ExceptPluginsList) > 0
self.scanner = Scanner(self.Core)
self.InitExecutionRegistry()
self.showOutput = True
def __init__(self, CoreObj, Options):
self.Core = CoreObj
#This should be dynamic from filesystem:
#self.PluginGroups = [ 'web', 'net', 'aux' ]
#self.PluginTypes = [ 'passive', 'semi_passive', 'active', 'grep' ]
#self.AllowedPluginTypes = self.GetAllowedPluginTypes(Options['PluginType'].split(','))
#self.Simulation, self.Scope, self.PluginGroup, self.Algorithm, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['Algorithm'], Options['ListPlugins'] ]
self.Simulation, self.Scope, self.PluginGroup, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['ListPlugins'] ]
self.OnlyPluginsList = self.ValidateAndFormatPluginList(Options['OnlyPlugins'])
self.ExceptPluginsList = self.ValidateAndFormatPluginList(Options['ExceptPlugins'])
#print "OnlyPlugins="+str(self.OnlyPluginsList)
#print "ExceptPlugins="+str(self.ExceptPluginsList)
#print "Options['PluginType']="+str(Options['PluginType'])
if isinstance(Options['PluginType'], str): # For special plugin types like "quiet" -> "semi_passive" + "passive"
Options['PluginType'] = Options['PluginType'].split(',')
self.AllowedPlugins = self.Core.DB.Plugin.GetPluginsByGroupType(self.PluginGroup, Options['PluginType'])
self.OnlyPluginsSet = len(self.OnlyPluginsList) > 0
self.ExceptPluginsSet = len(self.ExceptPluginsList) > 0
self.scanner = Scanner(self.Core)
self.InitExecutionRegistry()
self.showOutput = True
def init(self, options):
self.init_options(options)
self.Core = self.get_component("core")
self.plugin_output = self.get_component("plugin_output")
self.reporter = self.get_component("reporter")
self.scanner = Scanner()
class PluginHandler(BaseComponent, PluginHandlerInterface):
COMPONENT_NAME = "plugin_handler"
def __init__(self, options):
self.register_in_service_locator()
self.Core = None
self.db = self.get_component("db")
self.config = self.get_component("config")
self.plugin_output = None
self.db_plugin = self.get_component("db_plugin")
self.target = self.get_component("target")
self.transaction = self.get_component("transaction")
self.error_handler = self.get_component("error_handler")
self.reporter = None
self.timer = self.get_component("timer")
self.init_options(options)
def init_options(self, options):
"""Initialize CLI options for each instance of PluginHandler."""
self.PluginCount = 0
self.Simulation = options['Simulation']
self.Scope = options['Scope']
self.PluginGroup = options['PluginGroup']
self.OnlyPluginsList = self.ValidateAndFormatPluginList(options.get('OnlyPlugins'))
self.ExceptPluginsList = self.ValidateAndFormatPluginList(options.get('ExceptPlugins'))
if isinstance(options.get('PluginType'), str): # For special plugin types like "quiet" -> "semi_passive" + "passive"
options['PluginType'] = options['PluginType'].split(',')
self.scanner = None
self.InitExecutionRegistry()
def init(self, options):
self.init_options(options)
self.Core = self.get_component("core")
self.plugin_output = self.get_component("plugin_output")
self.reporter = self.get_component("reporter")
self.scanner = Scanner()
def PluginAlreadyRun(self, PluginInfo):
return self.plugin_output.PluginAlreadyRun(PluginInfo)
def ValidateAndFormatPluginList(self, plugin_codes):
"""Validate the plugin codes by checking if they exist.
:param list plugin_codes: OWTF plugin codes to be validated.
:return: validated plugin codes.
:rtype: list
"""
# Ensure there is always a list to iterate from! :)
if not plugin_codes:
return []
valid_plugin_codes = []
plugins_by_group = self.db_plugin.GetPluginsByGroup(self.PluginGroup)
for code in plugin_codes:
found = False
for plugin in plugins_by_group: # Processing Loop
if code in [plugin['code'], plugin['name']]:
valid_plugin_codes.append(plugin['code'])
found = True
break
if not found:
self.error_handler.FrameworkAbort(
"The code '%s' is not a valid plugin, please use the -l option to see available plugin names and codes" % code),
return valid_plugin_codes # Return list of Codes
def InitExecutionRegistry(self):
# Initialises the Execution registry: As plugins execute they will be tracked here, useful to avoid calling plugins stupidly :)
self.ExecutionRegistry = defaultdict(list)
for Target in self.Scope:
self.ExecutionRegistry[Target] = []
def GetLastPluginExecution(self, Plugin):
ExecLog = self.ExecutionRegistry[
self.config.GetTarget()] # Get shorcut to relevant execution log for this target for readability below :)
NumItems = len(ExecLog)
if NumItems == 0:
return -1 # List is empty
for Index in range((NumItems - 1), -1, -1):
Match = True
# Compare all execution log values against the passed Plugin, if all match, return index to log record
for Key, Value in ExecLog[Index].items():
if not Key in Plugin or Plugin[Key] != Value:
Match = False
if Match:
return Index
return -1
def PluginAlreadyRun(self, PluginInfo):
return self.plugin_output.PluginAlreadyRun(PluginInfo)
def GetExecLogSinceLastExecution(self, Plugin):
# Get all execution entries from log since last time the passed plugin executed
return self.ExecutionRegistry[self.config.GetTarget()][self.GetLastPluginExecution(Plugin):]
def GetPluginOutputDir(self, Plugin):
# Organise results by OWASP Test type and then active, passive, semi_passive
if ((Plugin['group'] == 'web') or (Plugin['group'] == 'network')):
return os.path.join(self.target.GetPath('partial_url_output_path'), WipeBadCharsForFilename(Plugin['title']), Plugin['type'])
elif Plugin['group'] == 'auxiliary':
return os.path.join(self.config.Get('AUX_OUTPUT_PATH'), WipeBadCharsForFilename(Plugin['title']), Plugin['type'])
def RequestsPossible(self):
# Even passive plugins will make requests to external resources
return ['grep'] != self.db_plugin.GetTypesForGroup('web')
def DumpOutputFile(self, Filename, Contents, Plugin, RelativePath=False):
SaveDir = self.GetPluginOutputDir(Plugin)
abs_path = FileOperations.dump_file(Filename, Contents, SaveDir)
if RelativePath:
return (os.path.relpath(abs_path, self.config.GetOutputDirForTargets()))
return (abs_path)
def RetrieveAbsPath(self, RelativePath):
return (os.path.join(self.config.GetOutputDirForTargets(), RelativePath))
def exists(self, directory):
return os.path.exists(directory)
def GetModule(self, ModuleName, ModuleFile, ModulePath):
# Python fiddling to load a module from a file, there is probably a better way...
f, Filename, desc = imp.find_module(ModuleFile.split('.')[0], [ModulePath]) # ModulePath = os.path.abspath(ModuleFile)
return imp.load_module(ModuleName, f, Filename, desc)
def chosen_plugin(self, plugin, show_reason=False):
"""Verify that the plugin has been chosen by the user.
:param dict plugin: The plugin dictionary with all the information.
:param bool show_reason: If the plugin cannot be run, print the reason.
:return: True if the plugin has been chosen, False otherwise.
:rtype: bool
"""
chosen = True
reason = 'not-specified'
if plugin['group'] == self.PluginGroup:
# Skip plugins not present in the white-list defined by the user.
if self.OnlyPluginsList and plugin['code'] not in self.OnlyPluginsList:
chosen = False
reason = 'not in white-list'
# Skip plugins present in the black-list defined by the user.
if self.ExceptPluginsList and plugin['code'] in self.ExceptPluginsList:
chosen = False
reason = 'in black-list'
if plugin['type'] not in self.db_plugin.GetTypesForGroup(plugin['group']):
chosen = False # Skip plugin: Not matching selected type
reason = 'not matching selected type'
if not chosen and show_reason:
logging.warning(
'Plugin: %s (%s/%s) has not been chosen by the user (%s), skipping...',
plugin['title'],
plugin['group'],
plugin['type'],
reason)
return chosen
def force_overwrite(self):
# return self.config.Get('FORCE_OVERWRITE')
return False
def plugin_can_run(self, plugin, show_reason=False):
"""Verify that a plugin can be run by OWTF.
:param dict plugin: The plugin dictionary with all the information.
:param bool show_reason: If the plugin cannot be run, print the reason.
:return: True if the plugin can be run, False otherwise.
:rtype: bool
"""
if not self.chosen_plugin(plugin, show_reason=show_reason):
return False # Skip not chosen plugins
# Grep plugins to be always run and overwritten (they run once after semi_passive and then again after active):
if self.PluginAlreadyRun(plugin) and ((not self.force_overwrite() and not ('grep' == plugin['type'])) or plugin['type'] == 'external'):
if show_reason:
logging.warning(
"Plugin: %s (%s/%s) has already been run, skipping...",
plugin['title'],
plugin['group'],
plugin['type'])
return False
if 'grep' == plugin['type'] and self.PluginAlreadyRun(plugin):
# Grep plugins can only run if some active or semi_passive plugin was run since the last time
return False
return True
def GetPluginFullPath(self, PluginDir, Plugin):
return PluginDir + "/" + Plugin['type'] + "/" + Plugin['file'] # Path to run the plugin
def RunPlugin(self, PluginDir, Plugin, save_output=True):
PluginPath = self.GetPluginFullPath(PluginDir, Plugin)
(Path, Name) = os.path.split(PluginPath)
PluginOutput = self.GetModule("", Name, Path + "/").run(Plugin)
return PluginOutput
@staticmethod
def rank_plugin(output, pathname):
"""Rank the current plugin results using PTP.
Returns the ranking value.
"""
def extract_metasploit_modules(cmd):
"""Extract the metasploit modules contained in the plugin output.
Returns the list of (module name, output file) found, an empty list
otherwise.
"""
return [
(
output['output'].get('ModifiedCommand', '').split(' ')[3],
os.path.basename(
output['output'].get('RelativeFilePath', ''))
)
for output in cmd
if ('output' in output and
'metasploit' in output['output'].get('ModifiedCommand', ''))]
msf_modules = None
if output:
msf_modules = extract_metasploit_modules(output)
owtf_rank = -1 # Default ranking value set to Unknown.
try:
parser = PTP()
if msf_modules:
for module in msf_modules:
parser.parse(
pathname=pathname,
filename=module[1], # Path to output file.
plugin=module[0]) # Metasploit module name.
owtf_rank = max(owtf_rank, parser.get_highest_ranking())
else:
parser.parse(pathname=pathname)
owtf_rank = parser.get_highest_ranking()
except PTPError: # Not supported tool or report not found.
pass
if owtf_rank == UNKNOWN: # Ugly truth... PTP gives 0 for unranked but OWTF uses -1 instead...
owtf_rank = -1
return owtf_rank
def ProcessPlugin(self, plugin_dir, plugin, status={}):
"""Process a plugin from running to ranking.
:param str plugin_dir: Path to the plugin directory.
:param dict plugin: The plugin dictionary with all the information.
:param dict status: Running status of the plugin.
:return: The output generated by the plugin when run.
:return: None if the plugin was not run.
:rtype: list
"""
# Ensure that the plugin CAN be run before starting anything.
if not self.plugin_can_run(plugin, show_reason=True):
return None
# Save how long it takes for the plugin to run.
self.timer.start_timer('Plugin')
plugin['start'] = self.timer.get_start_date_time('Plugin')
# Use relative path from targets folders while saving
plugin['output_path'] = os.path.relpath(
self.GetPluginOutputDir(plugin),
self.config.GetOutputDirForTargets())
status['AllSkipped'] = False # A plugin is going to be run.
plugin['status'] = 'Running'
self.PluginCount += 1
logging.info(
'_' * 10 + ' %d - Target: %s -> Plugin: %s (%s/%s) ' + '_' * 10,
self.PluginCount,
self.target.GetTargetURL(),
plugin['title'],
plugin['group'],
plugin['type'])
# Skip processing in simulation mode, but show until line above
# to illustrate what will run
if self.Simulation:
return None
# DB empty => grep plugins will fail, skip!!
if ('grep' == plugin['type'] and self.transaction.NumTransactions() == 0):
logging.info(
'Skipped - Cannot run grep plugins: '
'The Transaction DB is empty')
return None
output = None
status_msg = ''
partial_output = []
abort_reason = ''
try:
output = self.RunPlugin(plugin_dir, plugin)
status_msg = 'Successful'
status['SomeSuccessful'] = True
except KeyboardInterrupt:
# Just explain why crashed.
status_msg = 'Aborted'
abort_reason = 'Aborted by User'
status['SomeAborted (Keyboard Interrupt)'] = True
except SystemExit:
# Abort plugin processing and get out to external exception
# handling, information saved elsewhere.
raise SystemExit
except PluginAbortException as PartialOutput:
status_msg = 'Aborted (by user)'
partial_output = PartialOutput.parameter
abort_reason = 'Aborted by User'
status['SomeAborted'] = True
except UnreachableTargetException as PartialOutput:
status_msg = 'Unreachable Target'
partial_output = PartialOutput.parameter
abort_reason = 'Unreachable Target'
status['SomeAborted'] = True
except FrameworkAbortException as PartialOutput:
status_msg = 'Aborted (Framework Exit)'
partial_output = PartialOutput.parameter
abort_reason = 'Framework Aborted'
# TODO: Handle this gracefully
# except:
# Plugin["status"] = "Crashed"
# cprint("Crashed")
# self.SavePluginInfo(self.Core.Error.Add("Plugin "+Plugin['Type']+"/"+Plugin['File']+" failed for target "+self.Core.Config.Get('TARGET')), Plugin) # Try to save something
# TODO: http://blog.tplus1.com/index.php/2007/09/28/the-python-logging-module-is-much-better-than-print-statements/
finally:
plugin['status'] = status_msg
plugin['end'] = self.timer.get_end_date_time('Plugin')
plugin['owtf_rank'] = self.rank_plugin(output, self.GetPluginOutputDir(plugin))
if status_msg == 'Successful':
self.plugin_output.SavePluginOutput(plugin, output)
else:
self.plugin_output.SavePartialPluginOutput(
plugin,
partial_output,
abort_reason)
if status_msg == 'Aborted':
self.error_handler.UserAbort('Plugin')
if abort_reason == 'Framework Aborted':
self.Core.finish()
return output
def ProcessPlugins(self):
status = {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True}
if self.PluginGroup in ['web', 'auxiliary', 'network']:
self.ProcessPluginsForTargetList(
self.PluginGroup,
status,
self.target.GetAll("ID"))
return status
def GetPluginGroupDir(self, PluginGroup):
PluginDir = self.config.FrameworkConfigGet('PLUGINS_DIR') + PluginGroup
return PluginDir
def SwitchToTarget(self, Target):
self.target.SetTarget(Target) # Tell Target DB that all Gets/Sets are now Target-specific
def get_plugins_in_order_for_PluginGroup(self, PluginGroup):
return self.db_plugin.GetOrder(PluginGroup)
def get_plugins_in_order(self, PluginGroup):
return self.db_plugin.GetOrder(PluginGroup)
def ProcessPluginsForTargetList(self, PluginGroup, Status, TargetList):
# TargetList param will be useful for netsec stuff to call this
PluginDir = self.GetPluginGroupDir(PluginGroup)
if PluginGroup == 'network':
portwaves = self.config.Get('PORTWAVES')
waves = portwaves.split(',')
waves.append('-1')
lastwave = 0
for Target in TargetList: # For each Target
self.scanner.scan_network(Target)
# Scanning and processing the first part of the ports
for i in range(1):
ports = self.config.GetTcpPorts(lastwave, waves[i])
print "probing for ports" + str(ports)
http = self.scanner.probe_network(Target, 'tcp', ports)
# Tell Config that all Gets/Sets are now
# Target-specific.
self.SwitchToTarget(Target)
for Plugin in self.get_plugins_in_order_for_PluginGroup(PluginGroup):
self.ProcessPlugin(PluginDir, Plugin, Status)
lastwave = waves[i]
for http_ports in http:
if http_ports == '443':
self.ProcessPluginsForTargetList(
'web', {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True},
{'https://' + Target.split('//')[1]}
)
else:
self.ProcessPluginsForTargetList(
'web', {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True},
{Target}
)
else:
pass
def clean_up(self):
if getattr(self, "WorkerManager", None) is not None:
self.WorkerManager.clean_up()
def SavePluginInfo(self, PluginOutput, Plugin):
self.db.SaveDBs() # Save new URLs to DB after each request
self.reporter.SavePluginReport(PluginOutput, Plugin) # Timer retrieved by Reporter
def show_plugin_list(self, group, msg=INTRO_BANNER_GENERAL):
if group == 'web':
logging.info(msg + INTRO_BANNER_WEB_PLUGIN_TYPE + "\nAvailable WEB plugins:")
elif group == 'auxiliary':
logging.info(msg + "\nAvailable AUXILIARY plugins:")
elif group == 'network':
logging.info(msg + "\nAvailable NETWORK plugins:")
for plugin_type in self.db_plugin.GetTypesForGroup(group):
self.show_plugin_types(plugin_type, group)
def show_plugin_types(self, plugin_type, group):
logging.info("\n" + '*' * 40 + " " + plugin_type.title().replace('_', '-') + " plugins " + '*' * 40)
for Plugin in self.db_plugin.GetPluginsByGroupType(group, plugin_type):
# 'Name' : PluginName, 'Code': PluginCode, 'File' : PluginFile, 'Descrip' : PluginDescrip } )
LineStart = " " + Plugin['type'] + ": " + Plugin['name']
Pad1 = "_" * (60 - len(LineStart))
Pad2 = "_" * (20 - len(Plugin['code']))
logging.info(LineStart + Pad1 + "(" + Plugin['code'] + ")" + Pad2 + Plugin['descrip'])
def init(self, options):
self.init_options(options)
self.Core = self.get_component("core")
self.plugin_output = self.get_component("plugin_output")
self.reporter = self.get_component("reporter")
self.scanner = Scanner()
class PluginHandler(BaseComponent, PluginHandlerInterface):
COMPONENT_NAME = "plugin_handler"
def __init__(self, options):
self.register_in_service_locator()
self.Core = None
self.db = self.get_component("db")
self.config = self.get_component("config")
self.plugin_output = None
self.db_plugin = self.get_component("db_plugin")
self.target = self.get_component("target")
self.transaction = self.get_component("transaction")
self.error_handler = self.get_component("error_handler")
self.reporter = None
self.timer = self.get_component("timer")
self.init_options(options)
def init_options(self, options):
"""Initialize CLI options for each instance of PluginHandler."""
self.PluginCount = 0
self.Simulation = options['Simulation']
self.Scope = options['Scope']
self.PluginGroup = options['PluginGroup']
self.OnlyPluginsList = self.ValidateAndFormatPluginList(options.get('OnlyPlugins'))
self.ExceptPluginsList = self.ValidateAndFormatPluginList(options.get('ExceptPlugins'))
# For special plugin types like "quiet" -> "semi_passive" + "passive"
if isinstance(options.get('PluginType'), str):
options['PluginType'] = options['PluginType'].split(',')
self.scanner = None
self.InitExecutionRegistry()
def init(self, options):
self.init_options(options)
self.Core = self.get_component("core")
self.plugin_output = self.get_component("plugin_output")
self.reporter = self.get_component("reporter")
self.scanner = Scanner()
def PluginAlreadyRun(self, PluginInfo):
return self.plugin_output.PluginAlreadyRun(PluginInfo)
def ValidateAndFormatPluginList(self, plugin_codes):
"""Validate the plugin codes by checking if they exist.
:param list plugin_codes: OWTF plugin codes to be validated.
:return: validated plugin codes.
:rtype: list
"""
# Ensure there is always a list to iterate from! :)
if not plugin_codes:
return []
valid_plugin_codes = []
plugins_by_group = self.db_plugin.GetPluginsByGroup(self.PluginGroup)
for code in plugin_codes:
found = False
for plugin in plugins_by_group: # Processing Loop
if code in [plugin['code'], plugin['name']]:
valid_plugin_codes.append(plugin['code'])
found = True
break
if not found:
self.error_handler.FrameworkAbort("The code '%s' is not a valid plugin, please use the -l option to see"
"available plugin names and codes" % code)
return valid_plugin_codes # Return list of Codes
def InitExecutionRegistry(self):
# Initialises the Execution registry: As plugins execute they will be tracked here
# Useful to avoid calling plugins stupidly :)
self.ExecutionRegistry = defaultdict(list)
for Target in self.Scope:
self.ExecutionRegistry[Target] = []
def GetLastPluginExecution(self, Plugin):
ExecLog = self.ExecutionRegistry[
self.config.GetTarget()] # Get shorcut to relevant execution log for this target for readability below :)
NumItems = len(ExecLog)
if NumItems == 0:
return -1 # List is empty
for Index in range((NumItems - 1), -1, -1):
Match = True
# Compare all execution log values against the passed Plugin, if all match, return index to log record
for Key, Value in ExecLog[Index].items():
if Key not in Plugin or Plugin[Key] != Value:
Match = False
if Match:
return Index
return -1
def GetExecLogSinceLastExecution(self, Plugin):
# Get all execution entries from log since last time the passed plugin executed
return self.ExecutionRegistry[self.config.GetTarget()][self.GetLastPluginExecution(Plugin):]
def GetPluginOutputDir(self, Plugin):
# Organise results by OWASP Test type and then active, passive, semi_passive
if ((Plugin['group'] == 'web') or (Plugin['group'] == 'network')):
return os.path.join(self.target.GetPath('partial_url_output_path'),
WipeBadCharsForFilename(Plugin['title']), Plugin['type'])
elif Plugin['group'] == 'auxiliary':
return os.path.join(self.config.FrameworkConfigGet('AUX_OUTPUT_PATH'), WipeBadCharsForFilename(Plugin['title']),
Plugin['type'])
def RequestsPossible(self):
# Even passive plugins will make requests to external resources
return ['grep'] != self.db_plugin.GetTypesForGroup('web')
def DumpOutputFile(self, Filename, Contents, Plugin, RelativePath=False):
SaveDir = self.GetPluginOutputDir(Plugin)
abs_path = FileOperations.dump_file(Filename, Contents, SaveDir)
if RelativePath:
return (os.path.relpath(abs_path, self.config.GetOutputDirForTargets()))
return (abs_path)
def RetrieveAbsPath(self, RelativePath):
return (os.path.join(self.config.GetOutputDirForTargets(), RelativePath))
def exists(self, directory):
return os.path.exists(directory)
def GetModule(self, ModuleName, ModuleFile, ModulePath):
# Python fiddling to load a module from a file, there is probably a better way...
# ModulePath = os.path.abspath(ModuleFile)
f, Filename, desc = imp.find_module(ModuleFile.split('.')[0], [ModulePath])
return imp.load_module(ModuleName, f, Filename, desc)
def chosen_plugin(self, plugin, show_reason=False):
"""Verify that the plugin has been chosen by the user.
:param dict plugin: The plugin dictionary with all the information.
:param bool show_reason: If the plugin cannot be run, print the reason.
:return: True if the plugin has been chosen, False otherwise.
:rtype: bool
"""
chosen = True
reason = 'not-specified'
if plugin['group'] == self.PluginGroup:
# Skip plugins not present in the white-list defined by the user.
if self.OnlyPluginsList and plugin['code'] not in self.OnlyPluginsList:
chosen = False
reason = 'not in white-list'
# Skip plugins present in the black-list defined by the user.
if self.ExceptPluginsList and plugin['code'] in self.ExceptPluginsList:
chosen = False
reason = 'in black-list'
if plugin['type'] not in self.db_plugin.GetTypesForGroup(plugin['group']):
chosen = False # Skip plugin: Not matching selected type
reason = 'not matching selected type'
if not chosen and show_reason:
logging.warning(
'Plugin: %s (%s/%s) has not been chosen by the user (%s), skipping...',
plugin['title'],
plugin['group'],
plugin['type'],
reason)
return chosen
def force_overwrite(self):
return self.config.Get('FORCE_OVERWRITE')
def plugin_can_run(self, plugin, show_reason=False):
"""Verify that a plugin can be run by OWTF.
:param dict plugin: The plugin dictionary with all the information.
:param bool show_reason: If the plugin cannot be run, print the reason.
:return: True if the plugin can be run, False otherwise.
:rtype: bool
"""
if not self.chosen_plugin(plugin, show_reason=show_reason):
return False # Skip not chosen plugins
# Grep plugins to be always run and overwritten (they run once after semi_passive and then again after active)
if self.PluginAlreadyRun(plugin) and ((not self.force_overwrite() and not ('grep' == plugin['type'])) or
plugin['type'] == 'external'):
if show_reason:
logging.warning(
"Plugin: %s (%s/%s) has already been run, skipping...",
plugin['title'],
plugin['group'],
plugin['type']
)
return False
if 'grep' == plugin['type'] and self.PluginAlreadyRun(plugin):
# Grep plugins can only run if some active or semi_passive plugin was run since the last time
return False
return True
def GetPluginFullPath(self, PluginDir, Plugin):
return "%s/%s/%s" % (PluginDir, Plugin['type'], Plugin['file']) # Path to run the plugin
def RunPlugin(self, PluginDir, Plugin, save_output=True):
PluginPath = self.GetPluginFullPath(PluginDir, Plugin)
(Path, Name) = os.path.split(PluginPath)
PluginOutput = self.GetModule("", Name, Path + "/").run(Plugin)
return PluginOutput
@staticmethod
def rank_plugin(output, pathname):
"""Rank the current plugin results using PTP.
Returns the ranking value.
"""
def extract_metasploit_modules(cmd):
"""Extract the metasploit modules contained in the plugin output.
Returns the list of (module name, output file) found, an empty list
otherwise.
"""
return [
(
output['output'].get('ModifiedCommand', '').split(' ')[3],
os.path.basename(output['output'].get('RelativeFilePath', ''))
)
for output in cmd
if ('output' in output and 'metasploit' in output['output'].get('ModifiedCommand', ''))]
msf_modules = None
if output:
msf_modules = extract_metasploit_modules(output)
owtf_rank = -1 # Default ranking value set to Unknown.
try:
parser = PTP()
if msf_modules:
for module in msf_modules:
# filename - Path to output file.
# plugin - Metasploit module name.
parser.parse(pathname=pathname, filename=module[1], plugin=module[0], light=True)
owtf_rank = max(owtf_rank, parser.highest_ranking)
else:
parser.parse(pathname=pathname, light=True)
owtf_rank = parser.highest_ranking
except PTPError: # Not supported tool or report not found.
pass
except Exception as e:
logging.error('Unexpected exception when running PTP: %s' % e)
if owtf_rank == UNKNOWN: # Ugly truth... PTP gives 0 for unranked but OWTF uses -1 instead...
owtf_rank = -1
return owtf_rank
def ProcessPlugin(self, plugin_dir, plugin, status={}):
"""Process a plugin from running to ranking.
:param str plugin_dir: Path to the plugin directory.
:param dict plugin: The plugin dictionary with all the information.
:param dict status: Running status of the plugin.
:return: The output generated by the plugin when run.
:return: None if the plugin was not run.
:rtype: list
"""
# Ensure that the plugin CAN be run before starting anything.
if not self.plugin_can_run(plugin, show_reason=True):
return None
# Save how long it takes for the plugin to run.
self.timer.start_timer('Plugin')
plugin['start'] = self.timer.get_start_date_time('Plugin')
# Use relative path from targets folders while saving
plugin['output_path'] = os.path.relpath(self.GetPluginOutputDir(plugin), self.config.GetOutputDirForTargets())
status['AllSkipped'] = False # A plugin is going to be run.
plugin['status'] = 'Running'
self.PluginCount += 1
logging.info(
'_' * 10 + ' %d - Target: %s -> Plugin: %s (%s/%s) ' + '_' * 10,
self.PluginCount,
self.target.GetTargetURL(),
plugin['title'],
plugin['group'],
plugin['type'])
# Skip processing in simulation mode, but show until line above
# to illustrate what will run
if self.Simulation:
return None
# DB empty => grep plugins will fail, skip!!
if ('grep' == plugin['type'] and self.transaction.NumTransactions() == 0):
logging.info('Skipped - Cannot run grep plugins: The Transaction DB is empty')
return None
output = None
status_msg = ''
partial_output = []
abort_reason = ''
try:
output = self.RunPlugin(plugin_dir, plugin)
status_msg = 'Successful'
status['SomeSuccessful'] = True
except KeyboardInterrupt:
# Just explain why crashed.
status_msg = 'Aborted'
abort_reason = 'Aborted by User'
status['SomeAborted (Keyboard Interrupt)'] = True
except SystemExit:
# Abort plugin processing and get out to external exception
# handling, information saved elsewhere.
raise SystemExit
except PluginAbortException as PartialOutput:
status_msg = 'Aborted (by user)'
partial_output = PartialOutput.parameter
abort_reason = 'Aborted by User'
status['SomeAborted'] = True
except UnreachableTargetException as PartialOutput:
status_msg = 'Unreachable Target'
partial_output = PartialOutput.parameter
abort_reason = 'Unreachable Target'
status['SomeAborted'] = True
except FrameworkAbortException as PartialOutput:
status_msg = 'Aborted (Framework Exit)'
partial_output = PartialOutput.parameter
abort_reason = 'Framework Aborted'
# TODO: Handle this gracefully
# Replace print by logging
finally:
plugin['status'] = status_msg
plugin['end'] = self.timer.get_end_date_time('Plugin')
plugin['owtf_rank'] = self.rank_plugin(output, self.GetPluginOutputDir(plugin))
try:
if status_msg == 'Successful':
self.plugin_output.SavePluginOutput(plugin, output)
else:
self.plugin_output.SavePartialPluginOutput(plugin, partial_output, abort_reason)
except SQLAlchemyError as e:
logging.error("Exception occurred while during database transaction : \n%s", str(e))
output += str(e)
if status_msg == 'Aborted':
self.error_handler.UserAbort('Plugin')
if abort_reason == 'Framework Aborted':
self.Core.finish()
return output
def ProcessPlugins(self):
status = {'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True}
if self.PluginGroup in ['web', 'auxiliary', 'network']:
self.ProcessPluginsForTargetList(self.PluginGroup, status, self.target.GetAll("ID"))
return status
def GetPluginGroupDir(self, PluginGroup):
PluginDir = self.config.FrameworkConfigGet('PLUGINS_DIR') + PluginGroup
return PluginDir
def SwitchToTarget(self, Target):
self.target.SetTarget(Target) # Tell Target DB that all Gets/Sets are now Target-specific
def ProcessPluginsForTargetList(self, PluginGroup, Status, TargetList):
# TargetList param will be useful for netsec stuff to call this
PluginDir = self.GetPluginGroupDir(PluginGroup)
if PluginGroup == 'network':
portwaves = self.config.Get('PORTWAVES')
waves = portwaves.split(',')
waves.append('-1')
lastwave = 0
for Target in TargetList: # For each Target
self.scanner.scan_network(Target)
# Scanning and processing the first part of the ports
for i in range(1):
ports = self.config.GetTcpPorts(lastwave, waves[i])
print "Probing for ports %s" % str(ports)
http = self.scanner.probe_network(Target, 'tcp', ports)
# Tell Config that all Gets/Sets are now
# Target-specific.
self.SwitchToTarget(Target)
for Plugin in PluginGroup:
self.ProcessPlugin(PluginDir, Plugin, Status)
lastwave = waves[i]
for http_ports in http:
if http_ports == '443':
self.ProcessPluginsForTargetList(
'web',
{'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True},
{'https://%s' % Target.split('//')[1]})
else:
self.ProcessPluginsForTargetList(
'web',
{'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True},
{Target})
else:
pass
def clean_up(self):
if getattr(self, "WorkerManager", None) is not None:
self.WorkerManager.clean_up()
def SavePluginInfo(self, PluginOutput, Plugin):
self.db.SaveDBs() # Save new URLs to DB after each request
self.reporter.SavePluginReport(PluginOutput, Plugin) # Timer retrieved by Reporter
def show_plugin_list(self, group, msg=INTRO_BANNER_GENERAL):
if group == 'web':
logging.info("%s%s\nAvailable WEB plugins:", msg, INTRO_BANNER_WEB_PLUGIN_TYPE)
elif group == 'auxiliary':
logging.info("%s\nAvailable AUXILIARY plugins:", msg)
elif group == 'network':
logging.info("%s\nAvailable NETWORK plugins:", msg)
for plugin_type in self.db_plugin.GetTypesForGroup(group):
self.show_plugin_types(plugin_type, group)
def show_plugin_types(self, plugin_type, group):
logging.info("\n%s %s plugins %s", '*' * 40, plugin_type.title().replace('_', '-'), '*' * 40)
for Plugin in self.db_plugin.GetPluginsByGroupType(group, plugin_type):
LineStart = " %s:%s" % (Plugin['type'], Plugin['name'])
Pad1 = "_" * (60 - len(LineStart))
Pad2 = "_" * (20 - len(Plugin['code']))
logging.info("%s%s(%s)%s%s", LineStart, Pad1, Plugin['code'], Pad2, Plugin['descrip'])
def before(self):
self._create_core_mock()
self.scanner = Scanner(self.core_mock)
class ScannerTests(BaseTestCase):
def before(self):
self._create_core_mock()
self.scanner = Scanner(self.core_mock)
def test_ping_sweep_with_full_scan_executes_nmap_and_grep(self):
nmap_regex = re.compile("nmap.*[-]PS.*")
grep_regex = re.compile("grep.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", grep_regex)
self.scanner.ping_sweep("target", "full")
def test_ping_sweep_with_arp_scan_executes_nmap_and_grep(self):
nmap_regex = re.compile("nmap.*[-]PR.*")
grep_regex = re.compile("grep.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", grep_regex)
self.scanner.ping_sweep("target", "arp")
def test_scan_and_grab_banners_with_tcp_uses_nmap_and_amap_for_fingerprinting(
self):
nmap_regex = re.compile("nmap.*[-](sV|sS).*[-](sV|sS).*")
amap_regex = re.compile("amap.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", amap_regex)
self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix",
"tcp", "")
def test_scan_and_grab_banners_with_udp_uses_nmap_and_amap_for_fingerprinting(
self):
nmap_regex = re.compile("nmap.*[-](sV|sU).*[-](sV|sU).*")
amap_regex = re.compile("amap.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", amap_regex)
self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix",
"udp", "")
def test_get_ports_for_service_returns_the_list_of_ports_associated_to_services(
self):
services = [
"snmp", "smb", "smtp", "ms-sql", "ftp", "X11", "ppp", "vnc",
"http-rpc-epmap", "msrpc", "http"
]
flexmock(self.scanner)
self.scanner.should_receive("get_nmap_services_file").and_return(
NMAP_SERVICES_FILE)
for service in services:
port_list = self.scanner.get_ports_for_service(service, "")
assert_that(isinstance(port_list, list))
assert_that(port_list is not None)
def test_target_service_scans_nmap_output_file(self):
file_lines = [
"Host: 127.0.0.1\tPorts: 7/filtered/tcp//echo//, 80/open/tcp//http/Microsoft IIS\t\n"
]
flexmock(self.scanner)
self.scanner.should_receive("open_file").and_return(
FileMock(file_lines))
self.scanner.target_service("nmap_file", "service")
def test_probe_service_for_hosts_sets_plugin_list_to_execute_and_returns_http_ports(
self):
flexmock(self.scanner)
self.scanner.should_receive("target_service").and_return(
"127.0.0.1:80")
self.core_mock.Config = flexmock()
self.core_mock.Config.should_receive("Set")
self.core_mock.PluginHandler = flexmock()
self.core_mock.PluginHandler.should_receive(
"ValidateAndFormatPluginList").once()
http_ports = self.scanner.probe_service_for_hosts(
"nmap_file", "target")
assert_that(isinstance(http_ports, list))
assert_that(http_ports, has_length(greater_than(0)))
def test_dns_sweep_looks_for_DNS_servers_and_abort_execution_if_no_domain_is_found(
self):
self._record_dns_sweep_first_steps()
flexmock(self.scanner)
self._stub_open_file(re.compile(".*\.dns_server.ips"),
["127.0.0.1\n", "127.0.0.1\n"])
self.scanner.should_receive("open_file").with_args(
re.compile(".*\.domain_names")).and_raise(IOError).once()
self.scanner.dns_sweep("file_with_ips.txt", "file_prefix")
def test_dns_sweep_looks_for_DNS_servers_and_tries_to_do_a_zone_transfer_on_found_domains(
self):
self._record_dns_sweep_first_steps()
flexmock(self.scanner)
self._stub_open_file(re.compile(".*\.dns_server.ips"),
["127.0.0.1\n", "127.0.0.1\n"])
self._stub_open_file(re.compile(".*\.domain_names"), ["domain1.com"])
self._mock_shell_method_with_args_once(
"shell_exec", re.compile("host [-]l.*")) # Retrieve domains
self._mock_shell_method_with_args_once(
"shell_exec", re.compile("wc\s[-]l.*cut.*"),
return_value=4) # Determines if succeeded
self._mock_shell_method_with_args_once(
"shell_exec", re.compile("rm\s[-]f\s.*\.axfr.*"))
self.scanner.dns_sweep("file_with_ips.txt", "file_prefix")
def _create_core_mock(self):
self.core_mock = flexmock()
self.core_mock.Shell = flexmock()
self._stub_shell_method("shell_exec", None)
def _stub_shell_method(self, method, expected_result):
self.core_mock.Shell.should_receive(method).and_return(expected_result)
def _mock_shell_method_with_args_once(self,
method,
args,
return_value=None):
if (return_value is None):
self.core_mock.Shell.should_receive(method).with_args(args).once()
else:
self.core_mock.Shell.should_receive(method).with_args(
args).and_return(return_value).once()
def _record_dns_sweep_first_steps(self):
nmap_dns_discovery_regex = re.compile("nmap.*[-]sS.*[-]p\s53.*")
grep_open_53_port_regex = re.compile("grep.*53/open")
rm_old_files_regex = re.compile("rm [-]f .*\.domain_names")
self._mock_shell_method_with_args_once("shell_exec",
nmap_dns_discovery_regex)
self._mock_shell_method_with_args_once("shell_exec",
grep_open_53_port_regex)
self._mock_shell_method_with_args_once("shell_exec",
rm_old_files_regex)
def _stub_open_file(self, args, file_lines):
returned_file = FileMock(file_lines)
self.scanner.should_receive("open_file").with_args(args).and_return(
returned_file)
class PluginHandler:
PluginCount = 0
def __init__(self, CoreObj, Options):
self.Core = CoreObj
#This should be dynamic from filesystem:
#self.PluginGroups = [ 'web', 'net', 'aux' ]
#self.PluginTypes = [ 'passive', 'semi_passive', 'active', 'grep' ]
#self.AllowedPluginTypes = self.GetAllowedPluginTypes(Options['PluginType'].split(','))
#self.Simulation, self.Scope, self.PluginGroup, self.Algorithm, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['Algorithm'], Options['ListPlugins'] ]
self.Simulation, self.Scope, self.PluginGroup, self.ListPlugins = [
Options['Simulation'], Options['Scope'], Options['PluginGroup'],
Options['ListPlugins']
]
self.OnlyPluginsList = self.ValidateAndFormatPluginList(
Options['OnlyPlugins'])
self.ExceptPluginsList = self.ValidateAndFormatPluginList(
Options['ExceptPlugins'])
#print "OnlyPlugins="+str(self.OnlyPluginsList)
#print "ExceptPlugins="+str(self.ExceptPluginsList)
#print "Options['PluginType']="+str(Options['PluginType'])
if isinstance(
Options['PluginType'], str
): # For special plugin types like "quiet" -> "semi_passive" + "passive"
Options['PluginType'] = Options['PluginType'].split(',')
self.AllowedPlugins = self.Core.DB.Plugin.GetPluginsByGroupType(
self.PluginGroup, Options['PluginType'])
self.OnlyPluginsSet = len(self.OnlyPluginsList) > 0
self.ExceptPluginsSet = len(self.ExceptPluginsList) > 0
self.scanner = Scanner(self.Core)
self.InitExecutionRegistry()
self.showOutput = True
def ValidateAndFormatPluginList(self, PluginList):
List = [] # Ensure there is always a list to iterate from! :)
if PluginList != None:
List = PluginList
ValidatedList = []
#print "List to validate="+str(List)
for Item in List:
Found = False
for Plugin in self.Core.DB.Plugin.GetPluginsByGroup(
self.PluginGroup): # Processing Loop
if Item in [Plugin['code'], Plugin['name']]:
ValidatedList.append(Plugin['code'])
Found = True
break
if not Found:
cprint(
"ERROR: The code '" + Item +
"' is not a valid plugin, please use the -l option to see available plugin names and codes"
)
exit()
return ValidatedList # Return list of Codes
def InitExecutionRegistry(
self
): # Initialises the Execution registry: As plugins execute they will be tracked here, useful to avoid calling plugins stupidly :)
self.ExecutionRegistry = defaultdict(list)
for Target in self.Scope:
self.ExecutionRegistry[Target] = []
def GetLastPluginExecution(self, Plugin):
ExecLog = self.ExecutionRegistry[self.Core.Config.GetTarget(
)] # Get shorcut to relevant execution log for this target for readability below :)
NumItems = len(ExecLog)
#print "NumItems="+str(NumItems)
if NumItems == 0:
return -1 # List is empty
#print "NumItems="+str(NumItems)
#print str(ExecLog)
#print str(range((NumItems -1), 0))
for Index in range((NumItems - 1), -1, -1):
#print "Index="+str(Index)
#print str(ExecLog[Index])
Match = True
for Key, Value in ExecLog[Index].items(
): # Compare all execution log values against the passed Plugin, if all match, return index to log record
if not Key in Plugin or Plugin[Key] != Value:
Match = False
if Match:
#print str(PluginIprint "you have etered " + cnfo)+" was found!"
return Index
return -1
def PluginAlreadyRun(self, PluginInfo):
return self.Core.DB.POutput.PluginAlreadyRun(PluginInfo)
def GetExecLogSinceLastExecution(
self, Plugin
): # Get all execution entries from log since last time the passed plugin executed
return self.ExecutionRegistry[
self.Core.Config.GetTarget()][self.GetLastPluginExecution(Plugin):]
def NormalRequestsAllowed(self):
#AllowedPluginTypes = self.Core.Config.GetAllowedPluginTypes('web')
#GetAllowedPluginTypes('web')
AllowedPluginTypes = self.Core.Config.Plugin.GetAllowedTypes('web')
return 'semi_passive' in AllowedPluginTypes or 'active' in AllowedPluginTypes
def RequestsPossible(self):
# Even passive plugins will make requests to external resources
#return [ 'grep' ] != self.Core.Config.GetAllowedPluginTypes('web')
return ['grep'] != self.Core.DB.Plugin.GetTypesForGroup('web')
def DumpOutputFile(self, Filename, Contents, Plugin, RelativePath=False):
SaveDir = self.GetPluginOutputDir(Plugin)
abs_path = self.Core.DumpFile(Filename, Contents, SaveDir)
if RelativePath:
return (os.path.relpath(abs_path,
self.Core.Config.GetOutputDirForTargets()))
return (abs_path)
def RetrieveAbsPath(self, RelativePath):
return (os.path.join(self.Core.Config.GetOutputDirForTargets(),
RelativePath))
def GetPluginOutputDir(
self, Plugin
): # Organise results by OWASP Test type and then active, passive, semi_passive
#print "Plugin="+str(Plugin)+", Partial url ..="+str(self.Core.Config.Get('partial_url_output_path'))+", TARGET="+self.Core.Config.Get('TARGET')
if ((Plugin['group'] == 'web') or (Plugin['group'] == 'net')):
return os.path.join(
self.Core.DB.Target.GetPath('partial_url_output_path'),
WipeBadCharsForFilename(Plugin['title']), Plugin['type'])
elif Plugin['group'] == 'aux':
return os.path.join(self.Core.Config.Get('AUX_OUTPUT_PATH'),
WipeBadCharsForFilename(Plugin['title']),
Plugin['type'])
def exists(self, directory):
return os.path.exists(directory)
def GetModule(
self, ModuleName, ModuleFile, ModulePath
): # Python fiddling to load a module from a file, there is probably a better way...
f, Filename, desc = imp.find_module(
ModuleFile.split('.')[0],
[ModulePath]) #ModulePath = os.path.abspath(ModuleFile)
return imp.load_module(ModuleName, f, Filename, desc)
def IsChosenPlugin(self, Plugin):
Chosen = True
if Plugin['group'] == self.PluginGroup:
if self.OnlyPluginsSet and Plugin[
'code'] not in self.OnlyPluginsList:
Chosen = False # Skip plugins not present in the white-list defined by the user
if self.ExceptPluginsSet and Plugin[
'code'] in self.ExceptPluginsList:
Chosen = False # Skip plugins present in the black-list defined by the user
if Plugin['type'] not in self.Core.DB.Plugin.GetTypesForGroup(
Plugin['group']):
Chosen = False # Skip plugin: Not matching selected type
return Chosen
def IsActiveTestingPossible(
self
): # Checks if 1 active plugin is enabled = active testing possible:
Possible = False
#for PluginType, PluginFile, Title, Code, ReferenceURL in self.Core.Config.GetPlugins(): # Processing Loop
#for PluginType, PluginFile, Title, Code in self.Core.Config.Plugin.GetOrder(self.PluginGroup):
for Plugin in self.Core.Config.Plugin.GetOrder(self.PluginGroup):
if self.IsChosenPlugin(Plugin) and Plugin['type'] == 'active':
Possible = True
break
return Possible
def force_overwrite(self):
#return self.Core.Config.Get('FORCE_OVERWRITE')
return False
def CanPluginRun(self, Plugin, ShowMessages=False):
#if self.Core.IsTargetUnreachable():
# return False # Cannot run plugin if target is unreachable
if not self.IsChosenPlugin(Plugin):
return False # Skip not chosen plugins
# Grep plugins to be always run and overwritten (they run once after semi_passive and then again after active):
#if self.PluginAlreadyRun(Plugin) and not self.Core.Config.Get('FORCE_OVERWRITE'): #not Code == 'OWASP-WU-SPID': # For external plugin forced re-run (development)
if self.PluginAlreadyRun(Plugin) and (
(not self.force_overwrite() and not ('grep' == Plugin['type'])) or
Plugin['type'] == 'external'): #not Code == 'OWASP-WU-SPID':
if ShowMessages:
logging.info("Plugin: " + Plugin['title'] + " (" +
Plugin['type'] +
") has already been run, skipping ..")
#if Plugin['Type'] == 'external':
# External plugins are run only once per each run, so they are registered for all targets
# that are targets in that run. This is an alternative to chaning the js filters etc..
# self.register_plugin_for_all_targets(Plugin)
return False
if 'grep' == Plugin['type'] and self.PluginAlreadyRun(Plugin):
return False # Grep plugins can only run if some active or semi_passive plugin was run since the last time
return True
def GetPluginFullPath(self, PluginDir, Plugin):
return PluginDir + "/" + Plugin['type'] + "/" + Plugin[
'file'] # Path to run the plugin
def RunPlugin(self, PluginDir, Plugin, save_output=True):
PluginPath = self.GetPluginFullPath(PluginDir, Plugin)
(Path, Name) = os.path.split(PluginPath)
#(Name, Ext) = os.path.splitext(Name)
#self.Core.DB.Debug.Add("Running Plugin -> Plugin="+str(Plugin)+", PluginDir="+str(PluginDir))
PluginOutput = self.GetModule("", Name,
Path + "/").run(self.Core, Plugin)
#if save_output:
#print(PluginOutput)
#self.SavePluginInfo(PluginOutput, Plugin) # Timer retrieved here
return PluginOutput
@staticmethod
def rank_plugin(output, pathname):
"""Rank the current plugin results using PTP.
Returns the ranking value.
"""
def extract_metasploit_modules(cmd):
"""Extract the metasploit modules contained in the plugin output.
Returns the list of (module name, output file) found, an empty list
otherwise.
"""
return [
(output['output'].get('ModifiedCommand', '').split(' ')[3],
os.path.basename(output['output'].get('RelativeFilePath',
''))) for output in cmd
if ('output' in output and 'metasploit' in
output['output'].get('ModifiedCommand', ''))
]
msf_modules = None
if output: # Try to retrieve metasploit modules that were used.
msf_modules = extract_metasploit_modules(output)
owtf_rank = -1 # Default ranking value set to Unknown.
try:
parser = PTP()
if msf_modules: # PTP needs to know the msf module name.
for module in msf_modules:
parser.parse(
pathname=pathname,
filename=module[1], # Path to output file.
plugin=module[0]) # Metasploit module name.
owtf_rank = max(owtf_rank, parser.get_highest_ranking())
else: # Otherwise use the auto-detection mode.
parser.parse(pathname=pathname)
owtf_rank = parser.get_highest_ranking()
except PTPError: # Not supported tool or report not found.
pass
return owtf_rank
def ProcessPlugin(self, plugin_dir, plugin, status={}):
# Save how long it takes for the plugin to run.
self.Core.Timer.start_timer('Plugin')
plugin['start'] = self.Core.Timer.get_start_date_time('Plugin')
# Use relative path from targets folders while saving
plugin['output_path'] = os.path.relpath(
self.GetPluginOutputDir(plugin),
self.Core.Config.GetOutputDirForTargets())
status['AllSkipped'] = False # A plugin is going to be run.
plugin['status'] = 'Running'
self.PluginCount += 1
logging.info('_' * 10 + ' ' + str(self.PluginCount) + ' - Target: ' +
self.Core.DB.Target.GetTargetURL() + ' -> Plugin: ' +
plugin['title'] + ' (' + plugin['type'] + ') ' + '_' * 10)
# Skip processing in simulation mode, but show until line above
# to illustrate what will run
if self.Simulation:
return None
# DB empty => grep plugins will fail, skip!!
if ('grep' == plugin['type']
and self.Core.DB.Transaction.NumTransactions() == 0):
logging.info('Skipped - Cannot run grep plugins: '
'The Transaction DB is empty')
return None
output = None
status_msg = ''
partial_output = []
abort_reason = ''
try:
output = self.RunPlugin(plugin_dir, plugin)
status_msg = 'Successful'
status['SomeSuccessful'] = True
except KeyboardInterrupt:
# Just explain why crashed.
status_msg = 'Aborted'
abort_reason = 'Aborted by User'
status['SomeAborted (Keyboard Interrupt)'] = True
except SystemExit:
# Abort plugin processing and get out to external exception
# handling, information saved elsewhere.
raise SystemExit
except PluginAbortException as PartialOutput:
status_msg = 'Aborted (by user)'
partial_output = PartialOutput.parameter
abort_reason = 'Aborted by User'
status['SomeAborted'] = True
except UnreachableTargetException as PartialOutput:
status_msg = 'Unreachable Target'
partial_output = PartialOutput.parameter
abort_reason = 'Unreachable Target'
status['SomeAborted'] = True
except FrameworkAbortException as PartialOutput:
status_msg = 'Aborted (Framework Exit)'
partial_output = PartialOutput.parameter
abort_reason = 'Framework Aborted'
# TODO: Handle this gracefully
# except:
# Plugin["status"] = "Crashed"
# cprint("Crashed")
# self.SavePluginInfo(self.Core.Error.Add("Plugin "+Plugin['Type']+"/"+Plugin['File']+" failed for target "+self.Core.Config.Get('TARGET')), Plugin) # Try to save something
# TODO: http://blog.tplus1.com/index.php/2007/09/28/the-python-logging-module-is-much-better-than-print-statements/
finally:
plugin['status'] = status_msg
plugin['end'] = self.Core.Timer.get_end_date_time('Plugin')
plugin['owtf_rank'] = self.rank_plugin(
output, self.GetPluginOutputDir(plugin))
if status_msg == 'Successful':
self.Core.DB.POutput.SavePluginOutput(plugin, output)
else:
self.Core.DB.POutput.SavePartialPluginOutput(
plugin, partial_output, abort_reason)
if status_msg == 'Aborted':
self.Core.Error.UserAbort('Plugin')
if abort_reason == 'Framework Aborted':
self.Core.Finish('Aborted')
return output
def ProcessPlugins(self):
status = {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True
}
if self.PluginGroup in ['web', 'aux', 'net']:
self.ProcessPluginsForTargetList(self.PluginGroup, status,
self.Core.DB.Target.GetAll("id"))
return status
def GetPluginGroupDir(self, PluginGroup):
PluginDir = self.Core.Config.FrameworkConfigGet(
'PLUGINS_DIR') + PluginGroup
return PluginDir
def SwitchToTarget(self, Target):
self.Core.DB.Target.SetTarget(
Target
) # Tell Target DB that all Gets/Sets are now Target-specific
def get_plugins_in_order_for_PluginGroup(self, PluginGroup):
return self.Core.Config.Plugin.GetOrder(PluginGroup)
def get_plugins_in_order(self, PluginGroup):
return self.Core.Config.Plugin.GetOrder(PluginGroup)
def ProcessPluginsForTargetList(
self, PluginGroup, Status, TargetList
): # TargetList param will be useful for netsec stuff to call this
PluginDir = self.GetPluginGroupDir(PluginGroup)
if PluginGroup == 'net':
portwaves = self.Core.Config.Get('PORTWAVES')
waves = portwaves.split(',')
waves.append('-1')
lastwave = 0
for Target in TargetList: # For each Target
self.scanner.scan_network(Target)
#Scanning and processing the first part of the ports
for i in range(1):
ports = self.Core.Config.GetTcpPorts(lastwave, waves[i])
print "probing for ports" + str(ports)
http = self.scanner.probe_network(Target, 'tcp', ports)
# Tell Config that all Gets/Sets are now
# Target-specific.
self.SwitchToTarget(Target)
for Plugin in self.get_plugins_in_order_for_PluginGroup(
PluginGroup):
self.ProcessPlugin(PluginDir, Plugin, Status)
lastwave = waves[i]
for http_ports in http:
if http_ports == '443':
self.ProcessPluginsForTargetList(
'web', {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True
}, {'https://' + Target.split('//')[1]})
else:
self.ProcessPluginsForTargetList(
'web', {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True
}, {Target})
else:
pass
#self.WorkerManager.startinput()
#self.WorkerManager.fillWorkList(PluginGroup,TargetList)
#self.WorkerManager.spawn_workers()
#self.WorkerManager.manage_workers()
#self.WorkerManager.poisonPillToWorkers()
#Status = self.WorkerManager.joinWorker()
#if 'breadth' == self.Algorithm: # Loop plugins, then targets
# for Plugin in self.Core.Config.Plugin.GetOrder(PluginGroup):# For each Plugin
# #print "Processing Plugin="+str(Plugin)
# for Target in TargetList: # For each Target
# #print "Processing Target="+str(Target)
# self.SwitchToTarget(Target) # Tell Config that all Gets/Sets are now Target-specific
# self.ProcessPlugin( PluginDir, Plugin, Status )
#elif 'depth' == self.Algorithm: # Loop Targets, then plugins
# for Target in TargetList: # For each Target
# self.SwitchToTarget(Target) # Tell Config that all Gets/Sets are now Target-specific
# for Plugin in self.Core.Config.Plugin.GetOrder(PluginGroup):# For each Plugin
# self.ProcessPlugin( PluginDir, Plugin, Status )
def CleanUp(self):
self.WorkerManager.clean_up()
def SavePluginInfo(self, PluginOutput, Plugin):
self.Core.DB.SaveDBs() # Save new URLs to DB after each request
self.Core.Reporter.SavePluginReport(
PluginOutput, Plugin) # Timer retrieved by Reporter
def ShowPluginList(self):
if self.ListPlugins == 'web':
self.ShowWebPluginsBanner()
elif self.ListPlugins == 'aux':
self.ShowAuxPluginsBanner()
self.ShowPluginGroupPlugins(self.ListPlugins)
def ShowAuxPluginsBanner(self):
print(INTRO_BANNER_GENERAL + "\n Available AUXILIARY plugins:" "")
def ShowWebPluginsBanner(self):
print(INTRO_BANNER_GENERAL + INTRO_BANNER_WEB_PLUGIN_TYPE +
"\n Available WEB plugins:"
"")
def ShowPluginGroupPlugins(self, PluginGroup):
for PluginType in self.Core.Config.Plugin.GetTypesForGroup(
PluginGroup):
self.ShowPluginTypePlugins(PluginType, PluginGroup)
def ShowPluginTypePlugins(self, PluginType, PluginGroup):
cprint("\n" + '*' * 40 + " " + PluginType.title().replace('_', '-') +
" plugins " + '*' * 40)
for Plugin in self.Core.Config.Plugin.GetAll(PluginGroup, PluginType):
#'Name' : PluginName, 'Code': PluginCode, 'File' : PluginFile, 'Descrip' : PluginDescrip } )
LineStart = " " + Plugin['type'] + ": " + Plugin['name']
Pad1 = "_" * (60 - len(LineStart))
Pad2 = "_" * (20 - len(Plugin['code']))
cprint(LineStart + Pad1 + "(" + Plugin['code'] + ")" + Pad2 +
Plugin['descrip'])
def before(self):
self._create_core_mock()
self.scanner = Scanner(self.core_mock)
class ScannerTests(BaseTestCase):
def before(self):
self._create_core_mock()
self.scanner = Scanner(self.core_mock)
def test_ping_sweep_with_full_scan_executes_nmap_and_grep(self):
nmap_regex = re.compile("nmap.*[-]PS.*")
grep_regex = re.compile("grep.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", grep_regex)
self.scanner.ping_sweep("target", "full")
def test_ping_sweep_with_arp_scan_executes_nmap_and_grep(self):
nmap_regex = re.compile("nmap.*[-]PR.*")
grep_regex = re.compile("grep.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", grep_regex)
self.scanner.ping_sweep("target", "arp")
def test_scan_and_grab_banners_with_tcp_uses_nmap_and_amap_for_fingerprinting(self):
nmap_regex = re.compile("nmap.*[-](sV|sS).*[-](sV|sS).*")
amap_regex = re.compile("amap.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", amap_regex)
self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "tcp", "")
def test_scan_and_grab_banners_with_udp_uses_nmap_and_amap_for_fingerprinting(self):
nmap_regex = re.compile("nmap.*[-](sV|sU).*[-](sV|sU).*")
amap_regex = re.compile("amap.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", amap_regex)
self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "udp", "")
def test_get_ports_for_service_returns_the_list_of_ports_associated_to_services(self):
services = ["snmp", "smb", "smtp", "ms-sql", "ftp", "X11", "ppp", "vnc", "http-rpc-epmap", "msrpc", "http"]
flexmock(self.scanner)
self.scanner.should_receive("get_nmap_services_file").and_return(NMAP_SERVICES_FILE)
for service in services:
port_list = self.scanner.get_ports_for_service(service, "")
assert_that(isinstance(port_list, list))
assert_that(port_list is not None)
def test_target_service_scans_nmap_output_file(self):
file_lines = ["Host: 127.0.0.1\tPorts: 7/filtered/tcp//echo//, 80/open/tcp//http/Microsoft IIS\t\n"]
flexmock(self.scanner)
self.scanner.should_receive("open_file").and_return(FileMock(file_lines))
self.scanner.target_service("nmap_file", "service")
def test_probe_service_for_hosts_sets_plugin_list_to_execute_and_returns_http_ports(self):
flexmock(self.scanner)
self.scanner.should_receive("target_service").and_return("127.0.0.1:80")
self.core_mock.Config = flexmock()
self.core_mock.Config.should_receive("Set")
self.core_mock.PluginHandler = flexmock()
self.core_mock.PluginHandler.should_receive("ValidateAndFormatPluginList").once()
http_ports = self.scanner.probe_service_for_hosts("nmap_file", "target")
assert_that(isinstance(http_ports, list))
assert_that(http_ports, has_length(greater_than(0)))
def test_dns_sweep_looks_for_DNS_servers_and_abort_execution_if_no_domain_is_found(self):
self._record_dns_sweep_first_steps()
flexmock(self.scanner)
self._stub_open_file(re.compile(".*\.dns_server.ips"), ["127.0.0.1\n", "127.0.0.1\n"])
self.scanner.should_receive("open_file").with_args(re.compile(".*\.domain_names")).and_raise(IOError).once()
self.scanner.dns_sweep("file_with_ips.txt", "file_prefix")
def test_dns_sweep_looks_for_DNS_servers_and_tries_to_do_a_zone_transfer_on_found_domains(self):
self._record_dns_sweep_first_steps()
flexmock(self.scanner)
self._stub_open_file(re.compile(".*\.dns_server.ips"), ["127.0.0.1\n", "127.0.0.1\n"])
self._stub_open_file(re.compile(".*\.domain_names"), ["domain1.com"])
self._mock_shell_method_with_args_once("shell_exec", re.compile("host [-]l.*")) # Retrieve domains
self._mock_shell_method_with_args_once("shell_exec", re.compile("wc\s[-]l.*cut.*"), return_value=4) # Determines if succeeded
self._mock_shell_method_with_args_once("shell_exec", re.compile("rm\s[-]f\s.*\.axfr.*"))
self.scanner.dns_sweep("file_with_ips.txt", "file_prefix")
def _create_core_mock(self):
self.core_mock = flexmock()
self.core_mock.Shell = flexmock()
self._stub_shell_method("shell_exec", None)
def _stub_shell_method(self, method, expected_result):
self.core_mock.Shell.should_receive(method).and_return(expected_result)
def _mock_shell_method_with_args_once(self, method, args, return_value=None):
if (return_value is None):
self.core_mock.Shell.should_receive(method).with_args(args).once()
else:
self.core_mock.Shell.should_receive(method).with_args(args).and_return(return_value).once()
def _record_dns_sweep_first_steps(self):
nmap_dns_discovery_regex = re.compile("nmap.*[-]sS.*[-]p\s53.*")
grep_open_53_port_regex = re.compile("grep.*53/open")
rm_old_files_regex = re.compile("rm [-]f .*\.domain_names")
self._mock_shell_method_with_args_once("shell_exec", nmap_dns_discovery_regex)
self._mock_shell_method_with_args_once("shell_exec", grep_open_53_port_regex)
self._mock_shell_method_with_args_once("shell_exec", rm_old_files_regex)
def _stub_open_file(self, args, file_lines):
returned_file = FileMock(file_lines)
self.scanner.should_receive("open_file").with_args(args).and_return(returned_file)
class PluginHandler:
PluginCount = 0
def __init__(self, CoreObj, Options):
self.Core = CoreObj
#This should be dynamic from filesystem:
#self.PluginGroups = [ 'web', 'net', 'aux' ]
#self.PluginTypes = [ 'passive', 'semi_passive', 'active', 'grep' ]
#self.AllowedPluginTypes = self.GetAllowedPluginTypes(Options['PluginType'].split(','))
#self.Simulation, self.Scope, self.PluginGroup, self.Algorithm, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['Algorithm'], Options['ListPlugins'] ]
self.Simulation, self.Scope, self.PluginGroup, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['ListPlugins'] ]
self.OnlyPluginsList = self.ValidateAndFormatPluginList(Options['OnlyPlugins'])
self.ExceptPluginsList = self.ValidateAndFormatPluginList(Options['ExceptPlugins'])
#print "OnlyPlugins="+str(self.OnlyPluginsList)
#print "ExceptPlugins="+str(self.ExceptPluginsList)
#print "Options['PluginType']="+str(Options['PluginType'])
if isinstance(Options['PluginType'], str): # For special plugin types like "quiet" -> "semi_passive" + "passive"
Options['PluginType'] = Options['PluginType'].split(',')
self.AllowedPlugins = self.Core.DB.Plugin.GetPluginsByGroupType(self.PluginGroup, Options['PluginType'])
self.OnlyPluginsSet = len(self.OnlyPluginsList) > 0
self.ExceptPluginsSet = len(self.ExceptPluginsList) > 0
self.scanner = Scanner(self.Core)
self.InitExecutionRegistry()
self.showOutput = True
def ValidateAndFormatPluginList(self, PluginList):
List = [] # Ensure there is always a list to iterate from! :)
if PluginList != None:
List = PluginList
ValidatedList = []
#print "List to validate="+str(List)
for Item in List:
Found = False
for Plugin in self.Core.DB.Plugin.GetPluginsByGroup(self.PluginGroup): # Processing Loop
if Item in [ Plugin['code'], Plugin['name'] ]:
ValidatedList.append(Plugin['code'])
Found = True
break
if not Found:
cprint("ERROR: The code '"+Item+"' is not a valid plugin, please use the -l option to see available plugin names and codes")
exit()
return ValidatedList # Return list of Codes
def InitExecutionRegistry(self): # Initialises the Execution registry: As plugins execute they will be tracked here, useful to avoid calling plugins stupidly :)
self.ExecutionRegistry = defaultdict(list)
for Target in self.Scope:
self.ExecutionRegistry[Target] = []
def GetLastPluginExecution(self, Plugin):
ExecLog = self.ExecutionRegistry[self.Core.Config.GetTarget()] # Get shorcut to relevant execution log for this target for readability below :)
NumItems = len(ExecLog)
#print "NumItems="+str(NumItems)
if NumItems == 0:
return -1 # List is empty
#print "NumItems="+str(NumItems)
#print str(ExecLog)
#print str(range((NumItems -1), 0))
for Index in range((NumItems -1), -1, -1):
#print "Index="+str(Index)
#print str(ExecLog[Index])
Match = True
for Key, Value in ExecLog[Index].items(): # Compare all execution log values against the passed Plugin, if all match, return index to log record
if not Key in Plugin or Plugin[Key] != Value:
Match = False
if Match:
#print str(PluginIprint "you have etered " + cnfo)+" was found!"
return Index
return -1
def PluginAlreadyRun(self, PluginInfo):
return self.Core.DB.POutput.PluginAlreadyRun(PluginInfo)
def GetExecLogSinceLastExecution(self, Plugin): # Get all execution entries from log since last time the passed plugin executed
return self.ExecutionRegistry[self.Core.Config.GetTarget()][self.GetLastPluginExecution(Plugin):]
def NormalRequestsAllowed(self):
#AllowedPluginTypes = self.Core.Config.GetAllowedPluginTypes('web')
#GetAllowedPluginTypes('web')
AllowedPluginTypes = self.Core.Config.Plugin.GetAllowedTypes('web')
return 'semi_passive' in AllowedPluginTypes or 'active' in AllowedPluginTypes
def RequestsPossible(self):
# Even passive plugins will make requests to external resources
#return [ 'grep' ] != self.Core.Config.GetAllowedPluginTypes('web')
return [ 'grep' ] != self.Core.DB.Plugin.GetTypesForGroup('web')
def DumpOutputFile(self, Filename, Contents, Plugin, RelativePath=False):
SaveDir = self.GetPluginOutputDir(Plugin)
abs_path = self.Core.DumpFile(Filename, Contents, SaveDir)
if RelativePath:
return(os.path.relpath(abs_path, self.Core.Config.GetOutputDirForTargets()))
return(abs_path)
def RetrieveAbsPath(self, RelativePath):
return(os.path.join(self.Core.Config.GetOutputDirForTargets(), RelativePath))
def GetPluginOutputDir(self, Plugin): # Organise results by OWASP Test type and then active, passive, semi_passive
#print "Plugin="+str(Plugin)+", Partial url ..="+str(self.Core.Config.Get('partial_url_output_path'))+", TARGET="+self.Core.Config.Get('TARGET')
if ((Plugin['group'] == 'web') or (Plugin['group'] == 'net')):
return os.path.join(self.Core.DB.Target.GetPath('partial_url_output_path'), WipeBadCharsForFilename(Plugin['title']), Plugin['type'])
elif Plugin['group'] == 'aux':
return os.path.join(self.Core.Config.Get('AUX_OUTPUT_PATH'), WipeBadCharsForFilename(Plugin['title']), Plugin['type'])
def exists(self, directory):
return os.path.exists(directory)
def GetModule(self, ModuleName, ModuleFile, ModulePath):# Python fiddling to load a module from a file, there is probably a better way...
f, Filename, desc = imp.find_module(ModuleFile.split('.')[0], [ModulePath]) #ModulePath = os.path.abspath(ModuleFile)
return imp.load_module(ModuleName, f, Filename, desc)
def IsChosenPlugin(self, Plugin):
Chosen = True
if Plugin['group'] == self.PluginGroup:
if self.OnlyPluginsSet and Plugin['code'] not in self.OnlyPluginsList:
Chosen = False # Skip plugins not present in the white-list defined by the user
if self.ExceptPluginsSet and Plugin['code'] in self.ExceptPluginsList:
Chosen = False # Skip plugins present in the black-list defined by the user
if Plugin['type'] not in self.Core.DB.Plugin.GetTypesForGroup(Plugin['group']):
Chosen = False # Skip plugin: Not matching selected type
return Chosen
def IsActiveTestingPossible(self): # Checks if 1 active plugin is enabled = active testing possible:
Possible = False
#for PluginType, PluginFile, Title, Code, ReferenceURL in self.Core.Config.GetPlugins(): # Processing Loop
#for PluginType, PluginFile, Title, Code in self.Core.Config.Plugin.GetOrder(self.PluginGroup):
for Plugin in self.Core.Config.Plugin.GetOrder(self.PluginGroup):
if self.IsChosenPlugin(Plugin) and Plugin['type'] == 'active':
Possible = True
break
return Possible
def force_overwrite(self):
#return self.Core.Config.Get('FORCE_OVERWRITE')
return False
def CanPluginRun(self, Plugin, ShowMessages = False):
#if self.Core.IsTargetUnreachable():
# return False # Cannot run plugin if target is unreachable
if not self.IsChosenPlugin(Plugin):
return False # Skip not chosen plugins
# Grep plugins to be always run and overwritten (they run once after semi_passive and then again after active):
#if self.PluginAlreadyRun(Plugin) and not self.Core.Config.Get('FORCE_OVERWRITE'): #not Code == 'OWASP-WU-SPID': # For external plugin forced re-run (development)
if self.PluginAlreadyRun(Plugin) and ((not self.force_overwrite() and not ('grep' == Plugin['type'])) or Plugin['type'] == 'external'): #not Code == 'OWASP-WU-SPID':
if ShowMessages:
logging.info("Plugin: "+Plugin['title']+" ("+Plugin['type']+") has already been run, skipping ..")
#if Plugin['Type'] == 'external':
# External plugins are run only once per each run, so they are registered for all targets
# that are targets in that run. This is an alternative to chaning the js filters etc..
# self.register_plugin_for_all_targets(Plugin)
return False
if 'grep' == Plugin['type'] and self.PluginAlreadyRun(Plugin):
return False # Grep plugins can only run if some active or semi_passive plugin was run since the last time
return True
def GetPluginFullPath(self, PluginDir, Plugin):
return PluginDir+"/"+Plugin['type']+"/"+Plugin['file'] # Path to run the plugin
def RunPlugin(self, PluginDir, Plugin, save_output=True):
PluginPath = self.GetPluginFullPath(PluginDir, Plugin)
(Path, Name) = os.path.split(PluginPath)
#(Name, Ext) = os.path.splitext(Name)
#self.Core.DB.Debug.Add("Running Plugin -> Plugin="+str(Plugin)+", PluginDir="+str(PluginDir))
PluginOutput = self.GetModule("", Name, Path+"/").run(self.Core, Plugin)
#if save_output:
#print(PluginOutput)
#self.SavePluginInfo(PluginOutput, Plugin) # Timer retrieved here
return PluginOutput
@staticmethod
def rank_plugin(output, pathname):
"""Rank the current plugin results using PTP.
Returns the ranking value.
"""
def extract_metasploit_modules(cmd):
"""Extract the metasploit modules contained in the plugin output.
Returns the list of (module name, output file) found, an empty list
otherwise.
"""
return [
(
output['output'].get('ModifiedCommand', '').split(' ')[3],
os.path.basename(
output['output'].get('RelativeFilePath', ''))
)
for output in cmd
if ('output' in output and
'metasploit' in output['output'].get('ModifiedCommand', ''))]
msf_modules = None
if output: # Try to retrieve metasploit modules that were used.
msf_modules = extract_metasploit_modules(output)
owtf_rank = -1 # Default ranking value set to Unknown.
try:
parser = PTP()
if msf_modules: # PTP needs to know the msf module name.
for module in msf_modules:
parser.parse(
pathname=pathname,
filename=module[1], # Path to output file.
plugin=module[0]) # Metasploit module name.
owtf_rank = max(
owtf_rank,
parser.get_highest_ranking())
else: # Otherwise use the auto-detection mode.
parser.parse(pathname=pathname)
owtf_rank = parser.get_highest_ranking()
except PTPError: # Not supported tool or report not found.
pass
return owtf_rank
def ProcessPlugin(self, plugin_dir, plugin, status={}):
# Save how long it takes for the plugin to run.
self.Core.Timer.start_timer('Plugin')
plugin['start'] = self.Core.Timer.get_start_date_time('Plugin')
# Use relative path from targets folders while saving
plugin['output_path'] = os.path.relpath(
self.GetPluginOutputDir(plugin),
self.Core.Config.GetOutputDirForTargets())
status['AllSkipped'] = False # A plugin is going to be run.
plugin['status'] = 'Running'
self.PluginCount += 1
logging.info(
'_' * 10 + ' ' + str(self.PluginCount) +
' - Target: ' + self.Core.DB.Target.GetTargetURL() +
' -> Plugin: ' + plugin['title'] + ' (' +
plugin['type'] + ') ' + '_' * 10)
# Skip processing in simulation mode, but show until line above
# to illustrate what will run
if self.Simulation:
return None
# DB empty => grep plugins will fail, skip!!
if ('grep' == plugin['type'] and
self.Core.DB.Transaction.NumTransactions() == 0):
logging.info(
'Skipped - Cannot run grep plugins: '
'The Transaction DB is empty')
return None
output = None
status_msg = ''
partial_output = []
abort_reason = ''
try:
output = self.RunPlugin(plugin_dir, plugin)
status_msg = 'Successful'
status['SomeSuccessful'] = True
except KeyboardInterrupt:
# Just explain why crashed.
status_msg = 'Aborted'
abort_reason = 'Aborted by User'
status['SomeAborted (Keyboard Interrupt)'] = True
except SystemExit:
# Abort plugin processing and get out to external exception
# handling, information saved elsewhere.
raise SystemExit
except PluginAbortException as PartialOutput:
status_msg = 'Aborted (by user)'
partial_output = PartialOutput.parameter
abort_reason = 'Aborted by User'
status['SomeAborted'] = True
except UnreachableTargetException as PartialOutput:
status_msg = 'Unreachable Target'
partial_output = PartialOutput.parameter
abort_reason = 'Unreachable Target'
status['SomeAborted'] = True
except FrameworkAbortException as PartialOutput:
status_msg = 'Aborted (Framework Exit)'
partial_output = PartialOutput.parameter
abort_reason = 'Framework Aborted'
# TODO: Handle this gracefully
# except:
# Plugin["status"] = "Crashed"
# cprint("Crashed")
# self.SavePluginInfo(self.Core.Error.Add("Plugin "+Plugin['Type']+"/"+Plugin['File']+" failed for target "+self.Core.Config.Get('TARGET')), Plugin) # Try to save something
# TODO: http://blog.tplus1.com/index.php/2007/09/28/the-python-logging-module-is-much-better-than-print-statements/
finally:
plugin['status'] = status_msg
plugin['end'] = self.Core.Timer.get_end_date_time('Plugin')
plugin['owtf_rank'] = self.rank_plugin(
output,
self.GetPluginOutputDir(plugin))
if status_msg == 'Successful':
self.Core.DB.POutput.SavePluginOutput(plugin, output)
else:
self.Core.DB.POutput.SavePartialPluginOutput(
plugin,
partial_output,
abort_reason)
if status_msg == 'Aborted':
self.Core.Error.UserAbort('Plugin')
if abort_reason == 'Framework Aborted':
self.Core.Finish('Aborted')
return output
def ProcessPlugins(self):
status = {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True}
if self.PluginGroup in ['web', 'aux', 'net']:
self.ProcessPluginsForTargetList(
self.PluginGroup,
status,
self.Core.DB.Target.GetAll("id"))
return status
def GetPluginGroupDir(self, PluginGroup):
PluginDir = self.Core.Config.FrameworkConfigGet('PLUGINS_DIR')+PluginGroup
return PluginDir
def SwitchToTarget(self, Target):
self.Core.DB.Target.SetTarget(Target) # Tell Target DB that all Gets/Sets are now Target-specific
def get_plugins_in_order_for_PluginGroup(self, PluginGroup):
return self.Core.Config.Plugin.GetOrder(PluginGroup)
def get_plugins_in_order(self, PluginGroup):
return self.Core.Config.Plugin.GetOrder(PluginGroup)
def ProcessPluginsForTargetList(self, PluginGroup, Status, TargetList): # TargetList param will be useful for netsec stuff to call this
PluginDir = self.GetPluginGroupDir(PluginGroup)
if PluginGroup == 'net':
portwaves = self.Core.Config.Get('PORTWAVES')
waves = portwaves.split(',')
waves.append('-1')
lastwave=0
for Target in TargetList: # For each Target
self.scanner.scan_network(Target)
#Scanning and processing the first part of the ports
for i in range(1):
ports = self.Core.Config.GetTcpPorts(lastwave,waves[i])
print "probing for ports" + str(ports)
http = self.scanner.probe_network(Target, 'tcp', ports)
# Tell Config that all Gets/Sets are now
# Target-specific.
self.SwitchToTarget(Target)
for Plugin in self.get_plugins_in_order_for_PluginGroup(PluginGroup):
self.ProcessPlugin(PluginDir, Plugin, Status)
lastwave = waves[i]
for http_ports in http:
if http_ports == '443':
self.ProcessPluginsForTargetList(
'web', {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True},
{'https://' + Target.split('//')[1]}
)
else:
self.ProcessPluginsForTargetList(
'web', {
'SomeAborted': False,
'SomeSuccessful': False,
'AllSkipped': True},
{Target}
)
else:
pass
#self.WorkerManager.startinput()
#self.WorkerManager.fillWorkList(PluginGroup,TargetList)
#self.WorkerManager.spawn_workers()
#self.WorkerManager.manage_workers()
#self.WorkerManager.poisonPillToWorkers()
#Status = self.WorkerManager.joinWorker()
#if 'breadth' == self.Algorithm: # Loop plugins, then targets
# for Plugin in self.Core.Config.Plugin.GetOrder(PluginGroup):# For each Plugin
# #print "Processing Plugin="+str(Plugin)
# for Target in TargetList: # For each Target
# #print "Processing Target="+str(Target)
# self.SwitchToTarget(Target) # Tell Config that all Gets/Sets are now Target-specific
# self.ProcessPlugin( PluginDir, Plugin, Status )
#elif 'depth' == self.Algorithm: # Loop Targets, then plugins
# for Target in TargetList: # For each Target
# self.SwitchToTarget(Target) # Tell Config that all Gets/Sets are now Target-specific
# for Plugin in self.Core.Config.Plugin.GetOrder(PluginGroup):# For each Plugin
# self.ProcessPlugin( PluginDir, Plugin, Status )
def CleanUp(self):
self.WorkerManager.clean_up()
def SavePluginInfo(self, PluginOutput, Plugin):
self.Core.DB.SaveDBs() # Save new URLs to DB after each request
self.Core.Reporter.SavePluginReport(PluginOutput, Plugin) # Timer retrieved by Reporter
def ShowPluginList(self):
if self.ListPlugins == 'web':
self.ShowWebPluginsBanner()
elif self.ListPlugins == 'aux':
self.ShowAuxPluginsBanner()
self.ShowPluginGroupPlugins(self.ListPlugins)
def ShowAuxPluginsBanner(self):
print(INTRO_BANNER_GENERAL+"\n Available AUXILIARY plugins:""")
def ShowWebPluginsBanner(self):
print(INTRO_BANNER_GENERAL+INTRO_BANNER_WEB_PLUGIN_TYPE+"\n Available WEB plugins:""")
def ShowPluginGroupPlugins(self, PluginGroup):
for PluginType in self.Core.Config.Plugin.GetTypesForGroup(PluginGroup):
self.ShowPluginTypePlugins(PluginType,PluginGroup)
def ShowPluginTypePlugins(self, PluginType,PluginGroup):
cprint("\n"+'*' * 40+" "+PluginType.title().replace('_', '-')+" plugins "+'*' * 40)
for Plugin in self.Core.Config.Plugin.GetAll(PluginGroup, PluginType):
#'Name' : PluginName, 'Code': PluginCode, 'File' : PluginFile, 'Descrip' : PluginDescrip } )
LineStart = " "+Plugin['type']+": "+Plugin['name']
Pad1 = "_" * (60 - len(LineStart))
Pad2 = "_" * (20- len(Plugin['code']))
cprint(LineStart+Pad1+"("+Plugin['code']+")"+Pad2+Plugin['descrip'])
class ScannerTests(BaseTestCase):
def before(self):
self._create_core_mock()
self.scanner = Scanner(self.core_mock)
def test_ping_sweep_with_full_scan_executes_nmap_and_grep(self):
nmap_regex = re.compile("nmap.*[-]PS.*")
grep_regex = re.compile("grep.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", grep_regex)
self.scanner.ping_sweep("target", "full")
def test_ping_sweep_with_arp_scan_executes_nmap_and_grep(self):
nmap_regex = re.compile("nmap.*[-]PR.*")
grep_regex = re.compile("grep.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", grep_regex)
self.scanner.ping_sweep("target", "arp")
def test_scan_and_grab_banners_with_tcp_uses_nmap_and_amap_for_fingerprinting(self):
nmap_regex = re.compile("nmap.*[-](sV|sS).*[-](sV|sS).*")
amap_regex = re.compile("amap.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", amap_regex)
self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "tcp", "")
def test_scan_and_grab_banners_with_udp_uses_nmap_and_amap_for_fingerprinting(self):
nmap_regex = re.compile("nmap.*[-](sV|sU).*[-](sV|sU).*")
amap_regex = re.compile("amap.*")
self._mock_shell_method_with_args_once("shell_exec", nmap_regex)
self._mock_shell_method_with_args_once("shell_exec", amap_regex)
self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "udp", "")
def test_get_ports_for_service_returns_the_list_of_ports_associated_to_services(self):
services = ["snmp","smb","smtp","ms-sql","ftp","X11","ppp","vnc","http-rpc-epmap","msrpc","http"]
flexmock(self.scanner)
self.scanner.should_receive("get_nmap_services_file").and_return(NMAP_SERVICES_FILE)
for service in services:
port_list = self.scanner.get_ports_for_service(service, "")
assert_that(isinstance(port_list, list))
assert_that(port_list is not None)
def test_target_service_scans_nmap_output_file(self):
file_content = "Host: 127.0.0.1\tPorts: 7/filtered/tcp//echo//, 80/open/tcp//http/Microsoft IIS\t\n"
fake_file = flexmock()
fake_file.should_receive("read").and_return(file_content).once()
fake_file.should_receive("close").once()
flexmock(self.scanner)
self.scanner.should_receive("get_ports_for_service").and_return(["7", "80"])
self.scanner.should_receive("open_file").and_return(fake_file)
self.scanner.target_service("nmap_file", "service")
def test_probe_service_for_hosts_sets_plugin_list_to_execute_and_returns_http_ports(self):
flexmock(self.scanner)
self.scanner.should_receive("target_service").and_return("127.0.0.1:80")
self.core_mock.Config = flexmock()
self.core_mock.Config.should_receive("Set")
self.core_mock.PluginHandler = flexmock()
self.core_mock.PluginHandler.should_receive("ValidateAndFormatPluginList").once()
http_ports = self.scanner.probe_service_for_hosts("nmap_file", "target")
assert_that(isinstance(http_ports, list))
assert_that(http_ports, has_length(greater_than(0)))
def _create_core_mock(self):
self.core_mock = flexmock()
self.core_mock.Shell = flexmock()
self._stub_shell_method("shell_exec", None)
def _stub_shell_method(self, method, expected_result):
self.core_mock.Shell.should_receive(method).and_return(expected_result)
def _mock_shell_method_with_args_once(self, method, args):
self.core_mock.Shell.should_receive(method).with_args(args).once()
