def __init__(self, CoreObj, Options): self.Core = CoreObj #This should be dynamic from filesystem: #self.PluginGroups = [ 'web', 'net', 'aux' ] #self.PluginTypes = [ 'passive', 'semi_passive', 'active', 'grep' ] #self.AllowedPluginTypes = self.GetAllowedPluginTypes(Options['PluginType'].split(',')) #self.Simulation, self.Scope, self.PluginGroup, self.Algorithm, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['Algorithm'], Options['ListPlugins'] ] self.Simulation, self.Scope, self.PluginGroup, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['ListPlugins'] ] self.OnlyPluginsList = self.ValidateAndFormatPluginList( Options['OnlyPlugins']) self.ExceptPluginsList = self.ValidateAndFormatPluginList( Options['ExceptPlugins']) #print "OnlyPlugins="+str(self.OnlyPluginsList) #print "ExceptPlugins="+str(self.ExceptPluginsList) #print "Options['PluginType']="+str(Options['PluginType']) if isinstance( Options['PluginType'], str ): # For special plugin types like "quiet" -> "semi_passive" + "passive" Options['PluginType'] = Options['PluginType'].split(',') self.AllowedPlugins = self.Core.DB.Plugin.GetPluginsByGroupType( self.PluginGroup, Options['PluginType']) self.OnlyPluginsSet = len(self.OnlyPluginsList) > 0 self.ExceptPluginsSet = len(self.ExceptPluginsList) > 0 self.scanner = Scanner(self.Core) self.InitExecutionRegistry() self.showOutput = True
def __init__(self, CoreObj, Options): self.Core = CoreObj #This should be dynamic from filesystem: #self.PluginGroups = [ 'web', 'net', 'aux' ] #self.PluginTypes = [ 'passive', 'semi_passive', 'active', 'grep' ] #self.AllowedPluginTypes = self.GetAllowedPluginTypes(Options['PluginType'].split(',')) #self.Simulation, self.Scope, self.PluginGroup, self.Algorithm, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['Algorithm'], Options['ListPlugins'] ] self.Simulation, self.Scope, self.PluginGroup, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['ListPlugins'] ] self.OnlyPluginsList = self.ValidateAndFormatPluginList(Options['OnlyPlugins']) self.ExceptPluginsList = self.ValidateAndFormatPluginList(Options['ExceptPlugins']) #print "OnlyPlugins="+str(self.OnlyPluginsList) #print "ExceptPlugins="+str(self.ExceptPluginsList) #print "Options['PluginType']="+str(Options['PluginType']) if isinstance(Options['PluginType'], str): # For special plugin types like "quiet" -> "semi_passive" + "passive" Options['PluginType'] = Options['PluginType'].split(',') self.AllowedPlugins = self.Core.DB.Plugin.GetPluginsByGroupType(self.PluginGroup, Options['PluginType']) self.OnlyPluginsSet = len(self.OnlyPluginsList) > 0 self.ExceptPluginsSet = len(self.ExceptPluginsList) > 0 self.scanner = Scanner(self.Core) self.InitExecutionRegistry() self.showOutput = True
def init(self, options): self.init_options(options) self.Core = self.get_component("core") self.plugin_output = self.get_component("plugin_output") self.reporter = self.get_component("reporter") self.scanner = Scanner()
class PluginHandler(BaseComponent, PluginHandlerInterface): COMPONENT_NAME = "plugin_handler" def __init__(self, options): self.register_in_service_locator() self.Core = None self.db = self.get_component("db") self.config = self.get_component("config") self.plugin_output = None self.db_plugin = self.get_component("db_plugin") self.target = self.get_component("target") self.transaction = self.get_component("transaction") self.error_handler = self.get_component("error_handler") self.reporter = None self.timer = self.get_component("timer") self.init_options(options) def init_options(self, options): """Initialize CLI options for each instance of PluginHandler.""" self.PluginCount = 0 self.Simulation = options['Simulation'] self.Scope = options['Scope'] self.PluginGroup = options['PluginGroup'] self.OnlyPluginsList = self.ValidateAndFormatPluginList(options.get('OnlyPlugins')) self.ExceptPluginsList = self.ValidateAndFormatPluginList(options.get('ExceptPlugins')) if isinstance(options.get('PluginType'), str): # For special plugin types like "quiet" -> "semi_passive" + "passive" options['PluginType'] = options['PluginType'].split(',') self.scanner = None self.InitExecutionRegistry() def init(self, options): self.init_options(options) self.Core = self.get_component("core") self.plugin_output = self.get_component("plugin_output") self.reporter = self.get_component("reporter") self.scanner = Scanner() def PluginAlreadyRun(self, PluginInfo): return self.plugin_output.PluginAlreadyRun(PluginInfo) def ValidateAndFormatPluginList(self, plugin_codes): """Validate the plugin codes by checking if they exist. :param list plugin_codes: OWTF plugin codes to be validated. :return: validated plugin codes. :rtype: list """ # Ensure there is always a list to iterate from! :) if not plugin_codes: return [] valid_plugin_codes = [] plugins_by_group = self.db_plugin.GetPluginsByGroup(self.PluginGroup) for code in plugin_codes: found = False for plugin in plugins_by_group: # Processing Loop if code in [plugin['code'], plugin['name']]: valid_plugin_codes.append(plugin['code']) found = True break if not found: self.error_handler.FrameworkAbort( "The code '%s' is not a valid plugin, please use the -l option to see available plugin names and codes" % code), return valid_plugin_codes # Return list of Codes def InitExecutionRegistry(self): # Initialises the Execution registry: As plugins execute they will be tracked here, useful to avoid calling plugins stupidly :) self.ExecutionRegistry = defaultdict(list) for Target in self.Scope: self.ExecutionRegistry[Target] = [] def GetLastPluginExecution(self, Plugin): ExecLog = self.ExecutionRegistry[ self.config.GetTarget()] # Get shorcut to relevant execution log for this target for readability below :) NumItems = len(ExecLog) if NumItems == 0: return -1 # List is empty for Index in range((NumItems - 1), -1, -1): Match = True # Compare all execution log values against the passed Plugin, if all match, return index to log record for Key, Value in ExecLog[Index].items(): if not Key in Plugin or Plugin[Key] != Value: Match = False if Match: return Index return -1 def PluginAlreadyRun(self, PluginInfo): return self.plugin_output.PluginAlreadyRun(PluginInfo) def GetExecLogSinceLastExecution(self, Plugin): # Get all execution entries from log since last time the passed plugin executed return self.ExecutionRegistry[self.config.GetTarget()][self.GetLastPluginExecution(Plugin):] def GetPluginOutputDir(self, Plugin): # Organise results by OWASP Test type and then active, passive, semi_passive if ((Plugin['group'] == 'web') or (Plugin['group'] == 'network')): return os.path.join(self.target.GetPath('partial_url_output_path'), WipeBadCharsForFilename(Plugin['title']), Plugin['type']) elif Plugin['group'] == 'auxiliary': return os.path.join(self.config.Get('AUX_OUTPUT_PATH'), WipeBadCharsForFilename(Plugin['title']), Plugin['type']) def RequestsPossible(self): # Even passive plugins will make requests to external resources return ['grep'] != self.db_plugin.GetTypesForGroup('web') def DumpOutputFile(self, Filename, Contents, Plugin, RelativePath=False): SaveDir = self.GetPluginOutputDir(Plugin) abs_path = FileOperations.dump_file(Filename, Contents, SaveDir) if RelativePath: return (os.path.relpath(abs_path, self.config.GetOutputDirForTargets())) return (abs_path) def RetrieveAbsPath(self, RelativePath): return (os.path.join(self.config.GetOutputDirForTargets(), RelativePath)) def exists(self, directory): return os.path.exists(directory) def GetModule(self, ModuleName, ModuleFile, ModulePath): # Python fiddling to load a module from a file, there is probably a better way... f, Filename, desc = imp.find_module(ModuleFile.split('.')[0], [ModulePath]) # ModulePath = os.path.abspath(ModuleFile) return imp.load_module(ModuleName, f, Filename, desc) def chosen_plugin(self, plugin, show_reason=False): """Verify that the plugin has been chosen by the user. :param dict plugin: The plugin dictionary with all the information. :param bool show_reason: If the plugin cannot be run, print the reason. :return: True if the plugin has been chosen, False otherwise. :rtype: bool """ chosen = True reason = 'not-specified' if plugin['group'] == self.PluginGroup: # Skip plugins not present in the white-list defined by the user. if self.OnlyPluginsList and plugin['code'] not in self.OnlyPluginsList: chosen = False reason = 'not in white-list' # Skip plugins present in the black-list defined by the user. if self.ExceptPluginsList and plugin['code'] in self.ExceptPluginsList: chosen = False reason = 'in black-list' if plugin['type'] not in self.db_plugin.GetTypesForGroup(plugin['group']): chosen = False # Skip plugin: Not matching selected type reason = 'not matching selected type' if not chosen and show_reason: logging.warning( 'Plugin: %s (%s/%s) has not been chosen by the user (%s), skipping...', plugin['title'], plugin['group'], plugin['type'], reason) return chosen def force_overwrite(self): # return self.config.Get('FORCE_OVERWRITE') return False def plugin_can_run(self, plugin, show_reason=False): """Verify that a plugin can be run by OWTF. :param dict plugin: The plugin dictionary with all the information. :param bool show_reason: If the plugin cannot be run, print the reason. :return: True if the plugin can be run, False otherwise. :rtype: bool """ if not self.chosen_plugin(plugin, show_reason=show_reason): return False # Skip not chosen plugins # Grep plugins to be always run and overwritten (they run once after semi_passive and then again after active): if self.PluginAlreadyRun(plugin) and ((not self.force_overwrite() and not ('grep' == plugin['type'])) or plugin['type'] == 'external'): if show_reason: logging.warning( "Plugin: %s (%s/%s) has already been run, skipping...", plugin['title'], plugin['group'], plugin['type']) return False if 'grep' == plugin['type'] and self.PluginAlreadyRun(plugin): # Grep plugins can only run if some active or semi_passive plugin was run since the last time return False return True def GetPluginFullPath(self, PluginDir, Plugin): return PluginDir + "/" + Plugin['type'] + "/" + Plugin['file'] # Path to run the plugin def RunPlugin(self, PluginDir, Plugin, save_output=True): PluginPath = self.GetPluginFullPath(PluginDir, Plugin) (Path, Name) = os.path.split(PluginPath) PluginOutput = self.GetModule("", Name, Path + "/").run(Plugin) return PluginOutput @staticmethod def rank_plugin(output, pathname): """Rank the current plugin results using PTP. Returns the ranking value. """ def extract_metasploit_modules(cmd): """Extract the metasploit modules contained in the plugin output. Returns the list of (module name, output file) found, an empty list otherwise. """ return [ ( output['output'].get('ModifiedCommand', '').split(' ')[3], os.path.basename( output['output'].get('RelativeFilePath', '')) ) for output in cmd if ('output' in output and 'metasploit' in output['output'].get('ModifiedCommand', ''))] msf_modules = None if output: msf_modules = extract_metasploit_modules(output) owtf_rank = -1 # Default ranking value set to Unknown. try: parser = PTP() if msf_modules: for module in msf_modules: parser.parse( pathname=pathname, filename=module[1], # Path to output file. plugin=module[0]) # Metasploit module name. owtf_rank = max(owtf_rank, parser.get_highest_ranking()) else: parser.parse(pathname=pathname) owtf_rank = parser.get_highest_ranking() except PTPError: # Not supported tool or report not found. pass if owtf_rank == UNKNOWN: # Ugly truth... PTP gives 0 for unranked but OWTF uses -1 instead... owtf_rank = -1 return owtf_rank def ProcessPlugin(self, plugin_dir, plugin, status={}): """Process a plugin from running to ranking. :param str plugin_dir: Path to the plugin directory. :param dict plugin: The plugin dictionary with all the information. :param dict status: Running status of the plugin. :return: The output generated by the plugin when run. :return: None if the plugin was not run. :rtype: list """ # Ensure that the plugin CAN be run before starting anything. if not self.plugin_can_run(plugin, show_reason=True): return None # Save how long it takes for the plugin to run. self.timer.start_timer('Plugin') plugin['start'] = self.timer.get_start_date_time('Plugin') # Use relative path from targets folders while saving plugin['output_path'] = os.path.relpath( self.GetPluginOutputDir(plugin), self.config.GetOutputDirForTargets()) status['AllSkipped'] = False # A plugin is going to be run. plugin['status'] = 'Running' self.PluginCount += 1 logging.info( '_' * 10 + ' %d - Target: %s -> Plugin: %s (%s/%s) ' + '_' * 10, self.PluginCount, self.target.GetTargetURL(), plugin['title'], plugin['group'], plugin['type']) # Skip processing in simulation mode, but show until line above # to illustrate what will run if self.Simulation: return None # DB empty => grep plugins will fail, skip!! if ('grep' == plugin['type'] and self.transaction.NumTransactions() == 0): logging.info( 'Skipped - Cannot run grep plugins: ' 'The Transaction DB is empty') return None output = None status_msg = '' partial_output = [] abort_reason = '' try: output = self.RunPlugin(plugin_dir, plugin) status_msg = 'Successful' status['SomeSuccessful'] = True except KeyboardInterrupt: # Just explain why crashed. status_msg = 'Aborted' abort_reason = 'Aborted by User' status['SomeAborted (Keyboard Interrupt)'] = True except SystemExit: # Abort plugin processing and get out to external exception # handling, information saved elsewhere. raise SystemExit except PluginAbortException as PartialOutput: status_msg = 'Aborted (by user)' partial_output = PartialOutput.parameter abort_reason = 'Aborted by User' status['SomeAborted'] = True except UnreachableTargetException as PartialOutput: status_msg = 'Unreachable Target' partial_output = PartialOutput.parameter abort_reason = 'Unreachable Target' status['SomeAborted'] = True except FrameworkAbortException as PartialOutput: status_msg = 'Aborted (Framework Exit)' partial_output = PartialOutput.parameter abort_reason = 'Framework Aborted' # TODO: Handle this gracefully # except: # Plugin["status"] = "Crashed" # cprint("Crashed") # self.SavePluginInfo(self.Core.Error.Add("Plugin "+Plugin['Type']+"/"+Plugin['File']+" failed for target "+self.Core.Config.Get('TARGET')), Plugin) # Try to save something # TODO: http://blog.tplus1.com/index.php/2007/09/28/the-python-logging-module-is-much-better-than-print-statements/ finally: plugin['status'] = status_msg plugin['end'] = self.timer.get_end_date_time('Plugin') plugin['owtf_rank'] = self.rank_plugin(output, self.GetPluginOutputDir(plugin)) if status_msg == 'Successful': self.plugin_output.SavePluginOutput(plugin, output) else: self.plugin_output.SavePartialPluginOutput( plugin, partial_output, abort_reason) if status_msg == 'Aborted': self.error_handler.UserAbort('Plugin') if abort_reason == 'Framework Aborted': self.Core.finish() return output def ProcessPlugins(self): status = { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True} if self.PluginGroup in ['web', 'auxiliary', 'network']: self.ProcessPluginsForTargetList( self.PluginGroup, status, self.target.GetAll("ID")) return status def GetPluginGroupDir(self, PluginGroup): PluginDir = self.config.FrameworkConfigGet('PLUGINS_DIR') + PluginGroup return PluginDir def SwitchToTarget(self, Target): self.target.SetTarget(Target) # Tell Target DB that all Gets/Sets are now Target-specific def get_plugins_in_order_for_PluginGroup(self, PluginGroup): return self.db_plugin.GetOrder(PluginGroup) def get_plugins_in_order(self, PluginGroup): return self.db_plugin.GetOrder(PluginGroup) def ProcessPluginsForTargetList(self, PluginGroup, Status, TargetList): # TargetList param will be useful for netsec stuff to call this PluginDir = self.GetPluginGroupDir(PluginGroup) if PluginGroup == 'network': portwaves = self.config.Get('PORTWAVES') waves = portwaves.split(',') waves.append('-1') lastwave = 0 for Target in TargetList: # For each Target self.scanner.scan_network(Target) # Scanning and processing the first part of the ports for i in range(1): ports = self.config.GetTcpPorts(lastwave, waves[i]) print "probing for ports" + str(ports) http = self.scanner.probe_network(Target, 'tcp', ports) # Tell Config that all Gets/Sets are now # Target-specific. self.SwitchToTarget(Target) for Plugin in self.get_plugins_in_order_for_PluginGroup(PluginGroup): self.ProcessPlugin(PluginDir, Plugin, Status) lastwave = waves[i] for http_ports in http: if http_ports == '443': self.ProcessPluginsForTargetList( 'web', { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True}, {'https://' + Target.split('//')[1]} ) else: self.ProcessPluginsForTargetList( 'web', { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True}, {Target} ) else: pass def clean_up(self): if getattr(self, "WorkerManager", None) is not None: self.WorkerManager.clean_up() def SavePluginInfo(self, PluginOutput, Plugin): self.db.SaveDBs() # Save new URLs to DB after each request self.reporter.SavePluginReport(PluginOutput, Plugin) # Timer retrieved by Reporter def show_plugin_list(self, group, msg=INTRO_BANNER_GENERAL): if group == 'web': logging.info(msg + INTRO_BANNER_WEB_PLUGIN_TYPE + "\nAvailable WEB plugins:") elif group == 'auxiliary': logging.info(msg + "\nAvailable AUXILIARY plugins:") elif group == 'network': logging.info(msg + "\nAvailable NETWORK plugins:") for plugin_type in self.db_plugin.GetTypesForGroup(group): self.show_plugin_types(plugin_type, group) def show_plugin_types(self, plugin_type, group): logging.info("\n" + '*' * 40 + " " + plugin_type.title().replace('_', '-') + " plugins " + '*' * 40) for Plugin in self.db_plugin.GetPluginsByGroupType(group, plugin_type): # 'Name' : PluginName, 'Code': PluginCode, 'File' : PluginFile, 'Descrip' : PluginDescrip } ) LineStart = " " + Plugin['type'] + ": " + Plugin['name'] Pad1 = "_" * (60 - len(LineStart)) Pad2 = "_" * (20 - len(Plugin['code'])) logging.info(LineStart + Pad1 + "(" + Plugin['code'] + ")" + Pad2 + Plugin['descrip'])
class PluginHandler(BaseComponent, PluginHandlerInterface): COMPONENT_NAME = "plugin_handler" def __init__(self, options): self.register_in_service_locator() self.Core = None self.db = self.get_component("db") self.config = self.get_component("config") self.plugin_output = None self.db_plugin = self.get_component("db_plugin") self.target = self.get_component("target") self.transaction = self.get_component("transaction") self.error_handler = self.get_component("error_handler") self.reporter = None self.timer = self.get_component("timer") self.init_options(options) def init_options(self, options): """Initialize CLI options for each instance of PluginHandler.""" self.PluginCount = 0 self.Simulation = options['Simulation'] self.Scope = options['Scope'] self.PluginGroup = options['PluginGroup'] self.OnlyPluginsList = self.ValidateAndFormatPluginList(options.get('OnlyPlugins')) self.ExceptPluginsList = self.ValidateAndFormatPluginList(options.get('ExceptPlugins')) # For special plugin types like "quiet" -> "semi_passive" + "passive" if isinstance(options.get('PluginType'), str): options['PluginType'] = options['PluginType'].split(',') self.scanner = None self.InitExecutionRegistry() def init(self, options): self.init_options(options) self.Core = self.get_component("core") self.plugin_output = self.get_component("plugin_output") self.reporter = self.get_component("reporter") self.scanner = Scanner() def PluginAlreadyRun(self, PluginInfo): return self.plugin_output.PluginAlreadyRun(PluginInfo) def ValidateAndFormatPluginList(self, plugin_codes): """Validate the plugin codes by checking if they exist. :param list plugin_codes: OWTF plugin codes to be validated. :return: validated plugin codes. :rtype: list """ # Ensure there is always a list to iterate from! :) if not plugin_codes: return [] valid_plugin_codes = [] plugins_by_group = self.db_plugin.GetPluginsByGroup(self.PluginGroup) for code in plugin_codes: found = False for plugin in plugins_by_group: # Processing Loop if code in [plugin['code'], plugin['name']]: valid_plugin_codes.append(plugin['code']) found = True break if not found: self.error_handler.FrameworkAbort("The code '%s' is not a valid plugin, please use the -l option to see" "available plugin names and codes" % code) return valid_plugin_codes # Return list of Codes def InitExecutionRegistry(self): # Initialises the Execution registry: As plugins execute they will be tracked here # Useful to avoid calling plugins stupidly :) self.ExecutionRegistry = defaultdict(list) for Target in self.Scope: self.ExecutionRegistry[Target] = [] def GetLastPluginExecution(self, Plugin): ExecLog = self.ExecutionRegistry[ self.config.GetTarget()] # Get shorcut to relevant execution log for this target for readability below :) NumItems = len(ExecLog) if NumItems == 0: return -1 # List is empty for Index in range((NumItems - 1), -1, -1): Match = True # Compare all execution log values against the passed Plugin, if all match, return index to log record for Key, Value in ExecLog[Index].items(): if Key not in Plugin or Plugin[Key] != Value: Match = False if Match: return Index return -1 def GetExecLogSinceLastExecution(self, Plugin): # Get all execution entries from log since last time the passed plugin executed return self.ExecutionRegistry[self.config.GetTarget()][self.GetLastPluginExecution(Plugin):] def GetPluginOutputDir(self, Plugin): # Organise results by OWASP Test type and then active, passive, semi_passive if ((Plugin['group'] == 'web') or (Plugin['group'] == 'network')): return os.path.join(self.target.GetPath('partial_url_output_path'), WipeBadCharsForFilename(Plugin['title']), Plugin['type']) elif Plugin['group'] == 'auxiliary': return os.path.join(self.config.FrameworkConfigGet('AUX_OUTPUT_PATH'), WipeBadCharsForFilename(Plugin['title']), Plugin['type']) def RequestsPossible(self): # Even passive plugins will make requests to external resources return ['grep'] != self.db_plugin.GetTypesForGroup('web') def DumpOutputFile(self, Filename, Contents, Plugin, RelativePath=False): SaveDir = self.GetPluginOutputDir(Plugin) abs_path = FileOperations.dump_file(Filename, Contents, SaveDir) if RelativePath: return (os.path.relpath(abs_path, self.config.GetOutputDirForTargets())) return (abs_path) def RetrieveAbsPath(self, RelativePath): return (os.path.join(self.config.GetOutputDirForTargets(), RelativePath)) def exists(self, directory): return os.path.exists(directory) def GetModule(self, ModuleName, ModuleFile, ModulePath): # Python fiddling to load a module from a file, there is probably a better way... # ModulePath = os.path.abspath(ModuleFile) f, Filename, desc = imp.find_module(ModuleFile.split('.')[0], [ModulePath]) return imp.load_module(ModuleName, f, Filename, desc) def chosen_plugin(self, plugin, show_reason=False): """Verify that the plugin has been chosen by the user. :param dict plugin: The plugin dictionary with all the information. :param bool show_reason: If the plugin cannot be run, print the reason. :return: True if the plugin has been chosen, False otherwise. :rtype: bool """ chosen = True reason = 'not-specified' if plugin['group'] == self.PluginGroup: # Skip plugins not present in the white-list defined by the user. if self.OnlyPluginsList and plugin['code'] not in self.OnlyPluginsList: chosen = False reason = 'not in white-list' # Skip plugins present in the black-list defined by the user. if self.ExceptPluginsList and plugin['code'] in self.ExceptPluginsList: chosen = False reason = 'in black-list' if plugin['type'] not in self.db_plugin.GetTypesForGroup(plugin['group']): chosen = False # Skip plugin: Not matching selected type reason = 'not matching selected type' if not chosen and show_reason: logging.warning( 'Plugin: %s (%s/%s) has not been chosen by the user (%s), skipping...', plugin['title'], plugin['group'], plugin['type'], reason) return chosen def force_overwrite(self): return self.config.Get('FORCE_OVERWRITE') def plugin_can_run(self, plugin, show_reason=False): """Verify that a plugin can be run by OWTF. :param dict plugin: The plugin dictionary with all the information. :param bool show_reason: If the plugin cannot be run, print the reason. :return: True if the plugin can be run, False otherwise. :rtype: bool """ if not self.chosen_plugin(plugin, show_reason=show_reason): return False # Skip not chosen plugins # Grep plugins to be always run and overwritten (they run once after semi_passive and then again after active) if self.PluginAlreadyRun(plugin) and ((not self.force_overwrite() and not ('grep' == plugin['type'])) or plugin['type'] == 'external'): if show_reason: logging.warning( "Plugin: %s (%s/%s) has already been run, skipping...", plugin['title'], plugin['group'], plugin['type'] ) return False if 'grep' == plugin['type'] and self.PluginAlreadyRun(plugin): # Grep plugins can only run if some active or semi_passive plugin was run since the last time return False return True def GetPluginFullPath(self, PluginDir, Plugin): return "%s/%s/%s" % (PluginDir, Plugin['type'], Plugin['file']) # Path to run the plugin def RunPlugin(self, PluginDir, Plugin, save_output=True): PluginPath = self.GetPluginFullPath(PluginDir, Plugin) (Path, Name) = os.path.split(PluginPath) PluginOutput = self.GetModule("", Name, Path + "/").run(Plugin) return PluginOutput @staticmethod def rank_plugin(output, pathname): """Rank the current plugin results using PTP. Returns the ranking value. """ def extract_metasploit_modules(cmd): """Extract the metasploit modules contained in the plugin output. Returns the list of (module name, output file) found, an empty list otherwise. """ return [ ( output['output'].get('ModifiedCommand', '').split(' ')[3], os.path.basename(output['output'].get('RelativeFilePath', '')) ) for output in cmd if ('output' in output and 'metasploit' in output['output'].get('ModifiedCommand', ''))] msf_modules = None if output: msf_modules = extract_metasploit_modules(output) owtf_rank = -1 # Default ranking value set to Unknown. try: parser = PTP() if msf_modules: for module in msf_modules: # filename - Path to output file. # plugin - Metasploit module name. parser.parse(pathname=pathname, filename=module[1], plugin=module[0], light=True) owtf_rank = max(owtf_rank, parser.highest_ranking) else: parser.parse(pathname=pathname, light=True) owtf_rank = parser.highest_ranking except PTPError: # Not supported tool or report not found. pass except Exception as e: logging.error('Unexpected exception when running PTP: %s' % e) if owtf_rank == UNKNOWN: # Ugly truth... PTP gives 0 for unranked but OWTF uses -1 instead... owtf_rank = -1 return owtf_rank def ProcessPlugin(self, plugin_dir, plugin, status={}): """Process a plugin from running to ranking. :param str plugin_dir: Path to the plugin directory. :param dict plugin: The plugin dictionary with all the information. :param dict status: Running status of the plugin. :return: The output generated by the plugin when run. :return: None if the plugin was not run. :rtype: list """ # Ensure that the plugin CAN be run before starting anything. if not self.plugin_can_run(plugin, show_reason=True): return None # Save how long it takes for the plugin to run. self.timer.start_timer('Plugin') plugin['start'] = self.timer.get_start_date_time('Plugin') # Use relative path from targets folders while saving plugin['output_path'] = os.path.relpath(self.GetPluginOutputDir(plugin), self.config.GetOutputDirForTargets()) status['AllSkipped'] = False # A plugin is going to be run. plugin['status'] = 'Running' self.PluginCount += 1 logging.info( '_' * 10 + ' %d - Target: %s -> Plugin: %s (%s/%s) ' + '_' * 10, self.PluginCount, self.target.GetTargetURL(), plugin['title'], plugin['group'], plugin['type']) # Skip processing in simulation mode, but show until line above # to illustrate what will run if self.Simulation: return None # DB empty => grep plugins will fail, skip!! if ('grep' == plugin['type'] and self.transaction.NumTransactions() == 0): logging.info('Skipped - Cannot run grep plugins: The Transaction DB is empty') return None output = None status_msg = '' partial_output = [] abort_reason = '' try: output = self.RunPlugin(plugin_dir, plugin) status_msg = 'Successful' status['SomeSuccessful'] = True except KeyboardInterrupt: # Just explain why crashed. status_msg = 'Aborted' abort_reason = 'Aborted by User' status['SomeAborted (Keyboard Interrupt)'] = True except SystemExit: # Abort plugin processing and get out to external exception # handling, information saved elsewhere. raise SystemExit except PluginAbortException as PartialOutput: status_msg = 'Aborted (by user)' partial_output = PartialOutput.parameter abort_reason = 'Aborted by User' status['SomeAborted'] = True except UnreachableTargetException as PartialOutput: status_msg = 'Unreachable Target' partial_output = PartialOutput.parameter abort_reason = 'Unreachable Target' status['SomeAborted'] = True except FrameworkAbortException as PartialOutput: status_msg = 'Aborted (Framework Exit)' partial_output = PartialOutput.parameter abort_reason = 'Framework Aborted' # TODO: Handle this gracefully # Replace print by logging finally: plugin['status'] = status_msg plugin['end'] = self.timer.get_end_date_time('Plugin') plugin['owtf_rank'] = self.rank_plugin(output, self.GetPluginOutputDir(plugin)) try: if status_msg == 'Successful': self.plugin_output.SavePluginOutput(plugin, output) else: self.plugin_output.SavePartialPluginOutput(plugin, partial_output, abort_reason) except SQLAlchemyError as e: logging.error("Exception occurred while during database transaction : \n%s", str(e)) output += str(e) if status_msg == 'Aborted': self.error_handler.UserAbort('Plugin') if abort_reason == 'Framework Aborted': self.Core.finish() return output def ProcessPlugins(self): status = {'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True} if self.PluginGroup in ['web', 'auxiliary', 'network']: self.ProcessPluginsForTargetList(self.PluginGroup, status, self.target.GetAll("ID")) return status def GetPluginGroupDir(self, PluginGroup): PluginDir = self.config.FrameworkConfigGet('PLUGINS_DIR') + PluginGroup return PluginDir def SwitchToTarget(self, Target): self.target.SetTarget(Target) # Tell Target DB that all Gets/Sets are now Target-specific def ProcessPluginsForTargetList(self, PluginGroup, Status, TargetList): # TargetList param will be useful for netsec stuff to call this PluginDir = self.GetPluginGroupDir(PluginGroup) if PluginGroup == 'network': portwaves = self.config.Get('PORTWAVES') waves = portwaves.split(',') waves.append('-1') lastwave = 0 for Target in TargetList: # For each Target self.scanner.scan_network(Target) # Scanning and processing the first part of the ports for i in range(1): ports = self.config.GetTcpPorts(lastwave, waves[i]) print "Probing for ports %s" % str(ports) http = self.scanner.probe_network(Target, 'tcp', ports) # Tell Config that all Gets/Sets are now # Target-specific. self.SwitchToTarget(Target) for Plugin in PluginGroup: self.ProcessPlugin(PluginDir, Plugin, Status) lastwave = waves[i] for http_ports in http: if http_ports == '443': self.ProcessPluginsForTargetList( 'web', {'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True}, {'https://%s' % Target.split('//')[1]}) else: self.ProcessPluginsForTargetList( 'web', {'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True}, {Target}) else: pass def clean_up(self): if getattr(self, "WorkerManager", None) is not None: self.WorkerManager.clean_up() def SavePluginInfo(self, PluginOutput, Plugin): self.db.SaveDBs() # Save new URLs to DB after each request self.reporter.SavePluginReport(PluginOutput, Plugin) # Timer retrieved by Reporter def show_plugin_list(self, group, msg=INTRO_BANNER_GENERAL): if group == 'web': logging.info("%s%s\nAvailable WEB plugins:", msg, INTRO_BANNER_WEB_PLUGIN_TYPE) elif group == 'auxiliary': logging.info("%s\nAvailable AUXILIARY plugins:", msg) elif group == 'network': logging.info("%s\nAvailable NETWORK plugins:", msg) for plugin_type in self.db_plugin.GetTypesForGroup(group): self.show_plugin_types(plugin_type, group) def show_plugin_types(self, plugin_type, group): logging.info("\n%s %s plugins %s", '*' * 40, plugin_type.title().replace('_', '-'), '*' * 40) for Plugin in self.db_plugin.GetPluginsByGroupType(group, plugin_type): LineStart = " %s:%s" % (Plugin['type'], Plugin['name']) Pad1 = "_" * (60 - len(LineStart)) Pad2 = "_" * (20 - len(Plugin['code'])) logging.info("%s%s(%s)%s%s", LineStart, Pad1, Plugin['code'], Pad2, Plugin['descrip'])
def before(self): self._create_core_mock() self.scanner = Scanner(self.core_mock)
class ScannerTests(BaseTestCase): def before(self): self._create_core_mock() self.scanner = Scanner(self.core_mock) def test_ping_sweep_with_full_scan_executes_nmap_and_grep(self): nmap_regex = re.compile("nmap.*[-]PS.*") grep_regex = re.compile("grep.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", grep_regex) self.scanner.ping_sweep("target", "full") def test_ping_sweep_with_arp_scan_executes_nmap_and_grep(self): nmap_regex = re.compile("nmap.*[-]PR.*") grep_regex = re.compile("grep.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", grep_regex) self.scanner.ping_sweep("target", "arp") def test_scan_and_grab_banners_with_tcp_uses_nmap_and_amap_for_fingerprinting( self): nmap_regex = re.compile("nmap.*[-](sV|sS).*[-](sV|sS).*") amap_regex = re.compile("amap.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", amap_regex) self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "tcp", "") def test_scan_and_grab_banners_with_udp_uses_nmap_and_amap_for_fingerprinting( self): nmap_regex = re.compile("nmap.*[-](sV|sU).*[-](sV|sU).*") amap_regex = re.compile("amap.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", amap_regex) self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "udp", "") def test_get_ports_for_service_returns_the_list_of_ports_associated_to_services( self): services = [ "snmp", "smb", "smtp", "ms-sql", "ftp", "X11", "ppp", "vnc", "http-rpc-epmap", "msrpc", "http" ] flexmock(self.scanner) self.scanner.should_receive("get_nmap_services_file").and_return( NMAP_SERVICES_FILE) for service in services: port_list = self.scanner.get_ports_for_service(service, "") assert_that(isinstance(port_list, list)) assert_that(port_list is not None) def test_target_service_scans_nmap_output_file(self): file_lines = [ "Host: 127.0.0.1\tPorts: 7/filtered/tcp//echo//, 80/open/tcp//http/Microsoft IIS\t\n" ] flexmock(self.scanner) self.scanner.should_receive("open_file").and_return( FileMock(file_lines)) self.scanner.target_service("nmap_file", "service") def test_probe_service_for_hosts_sets_plugin_list_to_execute_and_returns_http_ports( self): flexmock(self.scanner) self.scanner.should_receive("target_service").and_return( "127.0.0.1:80") self.core_mock.Config = flexmock() self.core_mock.Config.should_receive("Set") self.core_mock.PluginHandler = flexmock() self.core_mock.PluginHandler.should_receive( "ValidateAndFormatPluginList").once() http_ports = self.scanner.probe_service_for_hosts( "nmap_file", "target") assert_that(isinstance(http_ports, list)) assert_that(http_ports, has_length(greater_than(0))) def test_dns_sweep_looks_for_DNS_servers_and_abort_execution_if_no_domain_is_found( self): self._record_dns_sweep_first_steps() flexmock(self.scanner) self._stub_open_file(re.compile(".*\.dns_server.ips"), ["127.0.0.1\n", "127.0.0.1\n"]) self.scanner.should_receive("open_file").with_args( re.compile(".*\.domain_names")).and_raise(IOError).once() self.scanner.dns_sweep("file_with_ips.txt", "file_prefix") def test_dns_sweep_looks_for_DNS_servers_and_tries_to_do_a_zone_transfer_on_found_domains( self): self._record_dns_sweep_first_steps() flexmock(self.scanner) self._stub_open_file(re.compile(".*\.dns_server.ips"), ["127.0.0.1\n", "127.0.0.1\n"]) self._stub_open_file(re.compile(".*\.domain_names"), ["domain1.com"]) self._mock_shell_method_with_args_once( "shell_exec", re.compile("host [-]l.*")) # Retrieve domains self._mock_shell_method_with_args_once( "shell_exec", re.compile("wc\s[-]l.*cut.*"), return_value=4) # Determines if succeeded self._mock_shell_method_with_args_once( "shell_exec", re.compile("rm\s[-]f\s.*\.axfr.*")) self.scanner.dns_sweep("file_with_ips.txt", "file_prefix") def _create_core_mock(self): self.core_mock = flexmock() self.core_mock.Shell = flexmock() self._stub_shell_method("shell_exec", None) def _stub_shell_method(self, method, expected_result): self.core_mock.Shell.should_receive(method).and_return(expected_result) def _mock_shell_method_with_args_once(self, method, args, return_value=None): if (return_value is None): self.core_mock.Shell.should_receive(method).with_args(args).once() else: self.core_mock.Shell.should_receive(method).with_args( args).and_return(return_value).once() def _record_dns_sweep_first_steps(self): nmap_dns_discovery_regex = re.compile("nmap.*[-]sS.*[-]p\s53.*") grep_open_53_port_regex = re.compile("grep.*53/open") rm_old_files_regex = re.compile("rm [-]f .*\.domain_names") self._mock_shell_method_with_args_once("shell_exec", nmap_dns_discovery_regex) self._mock_shell_method_with_args_once("shell_exec", grep_open_53_port_regex) self._mock_shell_method_with_args_once("shell_exec", rm_old_files_regex) def _stub_open_file(self, args, file_lines): returned_file = FileMock(file_lines) self.scanner.should_receive("open_file").with_args(args).and_return( returned_file)
class PluginHandler: PluginCount = 0 def __init__(self, CoreObj, Options): self.Core = CoreObj #This should be dynamic from filesystem: #self.PluginGroups = [ 'web', 'net', 'aux' ] #self.PluginTypes = [ 'passive', 'semi_passive', 'active', 'grep' ] #self.AllowedPluginTypes = self.GetAllowedPluginTypes(Options['PluginType'].split(',')) #self.Simulation, self.Scope, self.PluginGroup, self.Algorithm, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['Algorithm'], Options['ListPlugins'] ] self.Simulation, self.Scope, self.PluginGroup, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['ListPlugins'] ] self.OnlyPluginsList = self.ValidateAndFormatPluginList( Options['OnlyPlugins']) self.ExceptPluginsList = self.ValidateAndFormatPluginList( Options['ExceptPlugins']) #print "OnlyPlugins="+str(self.OnlyPluginsList) #print "ExceptPlugins="+str(self.ExceptPluginsList) #print "Options['PluginType']="+str(Options['PluginType']) if isinstance( Options['PluginType'], str ): # For special plugin types like "quiet" -> "semi_passive" + "passive" Options['PluginType'] = Options['PluginType'].split(',') self.AllowedPlugins = self.Core.DB.Plugin.GetPluginsByGroupType( self.PluginGroup, Options['PluginType']) self.OnlyPluginsSet = len(self.OnlyPluginsList) > 0 self.ExceptPluginsSet = len(self.ExceptPluginsList) > 0 self.scanner = Scanner(self.Core) self.InitExecutionRegistry() self.showOutput = True def ValidateAndFormatPluginList(self, PluginList): List = [] # Ensure there is always a list to iterate from! :) if PluginList != None: List = PluginList ValidatedList = [] #print "List to validate="+str(List) for Item in List: Found = False for Plugin in self.Core.DB.Plugin.GetPluginsByGroup( self.PluginGroup): # Processing Loop if Item in [Plugin['code'], Plugin['name']]: ValidatedList.append(Plugin['code']) Found = True break if not Found: cprint( "ERROR: The code '" + Item + "' is not a valid plugin, please use the -l option to see available plugin names and codes" ) exit() return ValidatedList # Return list of Codes def InitExecutionRegistry( self ): # Initialises the Execution registry: As plugins execute they will be tracked here, useful to avoid calling plugins stupidly :) self.ExecutionRegistry = defaultdict(list) for Target in self.Scope: self.ExecutionRegistry[Target] = [] def GetLastPluginExecution(self, Plugin): ExecLog = self.ExecutionRegistry[self.Core.Config.GetTarget( )] # Get shorcut to relevant execution log for this target for readability below :) NumItems = len(ExecLog) #print "NumItems="+str(NumItems) if NumItems == 0: return -1 # List is empty #print "NumItems="+str(NumItems) #print str(ExecLog) #print str(range((NumItems -1), 0)) for Index in range((NumItems - 1), -1, -1): #print "Index="+str(Index) #print str(ExecLog[Index]) Match = True for Key, Value in ExecLog[Index].items( ): # Compare all execution log values against the passed Plugin, if all match, return index to log record if not Key in Plugin or Plugin[Key] != Value: Match = False if Match: #print str(PluginIprint "you have etered " + cnfo)+" was found!" return Index return -1 def PluginAlreadyRun(self, PluginInfo): return self.Core.DB.POutput.PluginAlreadyRun(PluginInfo) def GetExecLogSinceLastExecution( self, Plugin ): # Get all execution entries from log since last time the passed plugin executed return self.ExecutionRegistry[ self.Core.Config.GetTarget()][self.GetLastPluginExecution(Plugin):] def NormalRequestsAllowed(self): #AllowedPluginTypes = self.Core.Config.GetAllowedPluginTypes('web') #GetAllowedPluginTypes('web') AllowedPluginTypes = self.Core.Config.Plugin.GetAllowedTypes('web') return 'semi_passive' in AllowedPluginTypes or 'active' in AllowedPluginTypes def RequestsPossible(self): # Even passive plugins will make requests to external resources #return [ 'grep' ] != self.Core.Config.GetAllowedPluginTypes('web') return ['grep'] != self.Core.DB.Plugin.GetTypesForGroup('web') def DumpOutputFile(self, Filename, Contents, Plugin, RelativePath=False): SaveDir = self.GetPluginOutputDir(Plugin) abs_path = self.Core.DumpFile(Filename, Contents, SaveDir) if RelativePath: return (os.path.relpath(abs_path, self.Core.Config.GetOutputDirForTargets())) return (abs_path) def RetrieveAbsPath(self, RelativePath): return (os.path.join(self.Core.Config.GetOutputDirForTargets(), RelativePath)) def GetPluginOutputDir( self, Plugin ): # Organise results by OWASP Test type and then active, passive, semi_passive #print "Plugin="+str(Plugin)+", Partial url ..="+str(self.Core.Config.Get('partial_url_output_path'))+", TARGET="+self.Core.Config.Get('TARGET') if ((Plugin['group'] == 'web') or (Plugin['group'] == 'net')): return os.path.join( self.Core.DB.Target.GetPath('partial_url_output_path'), WipeBadCharsForFilename(Plugin['title']), Plugin['type']) elif Plugin['group'] == 'aux': return os.path.join(self.Core.Config.Get('AUX_OUTPUT_PATH'), WipeBadCharsForFilename(Plugin['title']), Plugin['type']) def exists(self, directory): return os.path.exists(directory) def GetModule( self, ModuleName, ModuleFile, ModulePath ): # Python fiddling to load a module from a file, there is probably a better way... f, Filename, desc = imp.find_module( ModuleFile.split('.')[0], [ModulePath]) #ModulePath = os.path.abspath(ModuleFile) return imp.load_module(ModuleName, f, Filename, desc) def IsChosenPlugin(self, Plugin): Chosen = True if Plugin['group'] == self.PluginGroup: if self.OnlyPluginsSet and Plugin[ 'code'] not in self.OnlyPluginsList: Chosen = False # Skip plugins not present in the white-list defined by the user if self.ExceptPluginsSet and Plugin[ 'code'] in self.ExceptPluginsList: Chosen = False # Skip plugins present in the black-list defined by the user if Plugin['type'] not in self.Core.DB.Plugin.GetTypesForGroup( Plugin['group']): Chosen = False # Skip plugin: Not matching selected type return Chosen def IsActiveTestingPossible( self ): # Checks if 1 active plugin is enabled = active testing possible: Possible = False #for PluginType, PluginFile, Title, Code, ReferenceURL in self.Core.Config.GetPlugins(): # Processing Loop #for PluginType, PluginFile, Title, Code in self.Core.Config.Plugin.GetOrder(self.PluginGroup): for Plugin in self.Core.Config.Plugin.GetOrder(self.PluginGroup): if self.IsChosenPlugin(Plugin) and Plugin['type'] == 'active': Possible = True break return Possible def force_overwrite(self): #return self.Core.Config.Get('FORCE_OVERWRITE') return False def CanPluginRun(self, Plugin, ShowMessages=False): #if self.Core.IsTargetUnreachable(): # return False # Cannot run plugin if target is unreachable if not self.IsChosenPlugin(Plugin): return False # Skip not chosen plugins # Grep plugins to be always run and overwritten (they run once after semi_passive and then again after active): #if self.PluginAlreadyRun(Plugin) and not self.Core.Config.Get('FORCE_OVERWRITE'): #not Code == 'OWASP-WU-SPID': # For external plugin forced re-run (development) if self.PluginAlreadyRun(Plugin) and ( (not self.force_overwrite() and not ('grep' == Plugin['type'])) or Plugin['type'] == 'external'): #not Code == 'OWASP-WU-SPID': if ShowMessages: logging.info("Plugin: " + Plugin['title'] + " (" + Plugin['type'] + ") has already been run, skipping ..") #if Plugin['Type'] == 'external': # External plugins are run only once per each run, so they are registered for all targets # that are targets in that run. This is an alternative to chaning the js filters etc.. # self.register_plugin_for_all_targets(Plugin) return False if 'grep' == Plugin['type'] and self.PluginAlreadyRun(Plugin): return False # Grep plugins can only run if some active or semi_passive plugin was run since the last time return True def GetPluginFullPath(self, PluginDir, Plugin): return PluginDir + "/" + Plugin['type'] + "/" + Plugin[ 'file'] # Path to run the plugin def RunPlugin(self, PluginDir, Plugin, save_output=True): PluginPath = self.GetPluginFullPath(PluginDir, Plugin) (Path, Name) = os.path.split(PluginPath) #(Name, Ext) = os.path.splitext(Name) #self.Core.DB.Debug.Add("Running Plugin -> Plugin="+str(Plugin)+", PluginDir="+str(PluginDir)) PluginOutput = self.GetModule("", Name, Path + "/").run(self.Core, Plugin) #if save_output: #print(PluginOutput) #self.SavePluginInfo(PluginOutput, Plugin) # Timer retrieved here return PluginOutput @staticmethod def rank_plugin(output, pathname): """Rank the current plugin results using PTP. Returns the ranking value. """ def extract_metasploit_modules(cmd): """Extract the metasploit modules contained in the plugin output. Returns the list of (module name, output file) found, an empty list otherwise. """ return [ (output['output'].get('ModifiedCommand', '').split(' ')[3], os.path.basename(output['output'].get('RelativeFilePath', ''))) for output in cmd if ('output' in output and 'metasploit' in output['output'].get('ModifiedCommand', '')) ] msf_modules = None if output: # Try to retrieve metasploit modules that were used. msf_modules = extract_metasploit_modules(output) owtf_rank = -1 # Default ranking value set to Unknown. try: parser = PTP() if msf_modules: # PTP needs to know the msf module name. for module in msf_modules: parser.parse( pathname=pathname, filename=module[1], # Path to output file. plugin=module[0]) # Metasploit module name. owtf_rank = max(owtf_rank, parser.get_highest_ranking()) else: # Otherwise use the auto-detection mode. parser.parse(pathname=pathname) owtf_rank = parser.get_highest_ranking() except PTPError: # Not supported tool or report not found. pass return owtf_rank def ProcessPlugin(self, plugin_dir, plugin, status={}): # Save how long it takes for the plugin to run. self.Core.Timer.start_timer('Plugin') plugin['start'] = self.Core.Timer.get_start_date_time('Plugin') # Use relative path from targets folders while saving plugin['output_path'] = os.path.relpath( self.GetPluginOutputDir(plugin), self.Core.Config.GetOutputDirForTargets()) status['AllSkipped'] = False # A plugin is going to be run. plugin['status'] = 'Running' self.PluginCount += 1 logging.info('_' * 10 + ' ' + str(self.PluginCount) + ' - Target: ' + self.Core.DB.Target.GetTargetURL() + ' -> Plugin: ' + plugin['title'] + ' (' + plugin['type'] + ') ' + '_' * 10) # Skip processing in simulation mode, but show until line above # to illustrate what will run if self.Simulation: return None # DB empty => grep plugins will fail, skip!! if ('grep' == plugin['type'] and self.Core.DB.Transaction.NumTransactions() == 0): logging.info('Skipped - Cannot run grep plugins: ' 'The Transaction DB is empty') return None output = None status_msg = '' partial_output = [] abort_reason = '' try: output = self.RunPlugin(plugin_dir, plugin) status_msg = 'Successful' status['SomeSuccessful'] = True except KeyboardInterrupt: # Just explain why crashed. status_msg = 'Aborted' abort_reason = 'Aborted by User' status['SomeAborted (Keyboard Interrupt)'] = True except SystemExit: # Abort plugin processing and get out to external exception # handling, information saved elsewhere. raise SystemExit except PluginAbortException as PartialOutput: status_msg = 'Aborted (by user)' partial_output = PartialOutput.parameter abort_reason = 'Aborted by User' status['SomeAborted'] = True except UnreachableTargetException as PartialOutput: status_msg = 'Unreachable Target' partial_output = PartialOutput.parameter abort_reason = 'Unreachable Target' status['SomeAborted'] = True except FrameworkAbortException as PartialOutput: status_msg = 'Aborted (Framework Exit)' partial_output = PartialOutput.parameter abort_reason = 'Framework Aborted' # TODO: Handle this gracefully # except: # Plugin["status"] = "Crashed" # cprint("Crashed") # self.SavePluginInfo(self.Core.Error.Add("Plugin "+Plugin['Type']+"/"+Plugin['File']+" failed for target "+self.Core.Config.Get('TARGET')), Plugin) # Try to save something # TODO: http://blog.tplus1.com/index.php/2007/09/28/the-python-logging-module-is-much-better-than-print-statements/ finally: plugin['status'] = status_msg plugin['end'] = self.Core.Timer.get_end_date_time('Plugin') plugin['owtf_rank'] = self.rank_plugin( output, self.GetPluginOutputDir(plugin)) if status_msg == 'Successful': self.Core.DB.POutput.SavePluginOutput(plugin, output) else: self.Core.DB.POutput.SavePartialPluginOutput( plugin, partial_output, abort_reason) if status_msg == 'Aborted': self.Core.Error.UserAbort('Plugin') if abort_reason == 'Framework Aborted': self.Core.Finish('Aborted') return output def ProcessPlugins(self): status = { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True } if self.PluginGroup in ['web', 'aux', 'net']: self.ProcessPluginsForTargetList(self.PluginGroup, status, self.Core.DB.Target.GetAll("id")) return status def GetPluginGroupDir(self, PluginGroup): PluginDir = self.Core.Config.FrameworkConfigGet( 'PLUGINS_DIR') + PluginGroup return PluginDir def SwitchToTarget(self, Target): self.Core.DB.Target.SetTarget( Target ) # Tell Target DB that all Gets/Sets are now Target-specific def get_plugins_in_order_for_PluginGroup(self, PluginGroup): return self.Core.Config.Plugin.GetOrder(PluginGroup) def get_plugins_in_order(self, PluginGroup): return self.Core.Config.Plugin.GetOrder(PluginGroup) def ProcessPluginsForTargetList( self, PluginGroup, Status, TargetList ): # TargetList param will be useful for netsec stuff to call this PluginDir = self.GetPluginGroupDir(PluginGroup) if PluginGroup == 'net': portwaves = self.Core.Config.Get('PORTWAVES') waves = portwaves.split(',') waves.append('-1') lastwave = 0 for Target in TargetList: # For each Target self.scanner.scan_network(Target) #Scanning and processing the first part of the ports for i in range(1): ports = self.Core.Config.GetTcpPorts(lastwave, waves[i]) print "probing for ports" + str(ports) http = self.scanner.probe_network(Target, 'tcp', ports) # Tell Config that all Gets/Sets are now # Target-specific. self.SwitchToTarget(Target) for Plugin in self.get_plugins_in_order_for_PluginGroup( PluginGroup): self.ProcessPlugin(PluginDir, Plugin, Status) lastwave = waves[i] for http_ports in http: if http_ports == '443': self.ProcessPluginsForTargetList( 'web', { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True }, {'https://' + Target.split('//')[1]}) else: self.ProcessPluginsForTargetList( 'web', { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True }, {Target}) else: pass #self.WorkerManager.startinput() #self.WorkerManager.fillWorkList(PluginGroup,TargetList) #self.WorkerManager.spawn_workers() #self.WorkerManager.manage_workers() #self.WorkerManager.poisonPillToWorkers() #Status = self.WorkerManager.joinWorker() #if 'breadth' == self.Algorithm: # Loop plugins, then targets # for Plugin in self.Core.Config.Plugin.GetOrder(PluginGroup):# For each Plugin # #print "Processing Plugin="+str(Plugin) # for Target in TargetList: # For each Target # #print "Processing Target="+str(Target) # self.SwitchToTarget(Target) # Tell Config that all Gets/Sets are now Target-specific # self.ProcessPlugin( PluginDir, Plugin, Status ) #elif 'depth' == self.Algorithm: # Loop Targets, then plugins # for Target in TargetList: # For each Target # self.SwitchToTarget(Target) # Tell Config that all Gets/Sets are now Target-specific # for Plugin in self.Core.Config.Plugin.GetOrder(PluginGroup):# For each Plugin # self.ProcessPlugin( PluginDir, Plugin, Status ) def CleanUp(self): self.WorkerManager.clean_up() def SavePluginInfo(self, PluginOutput, Plugin): self.Core.DB.SaveDBs() # Save new URLs to DB after each request self.Core.Reporter.SavePluginReport( PluginOutput, Plugin) # Timer retrieved by Reporter def ShowPluginList(self): if self.ListPlugins == 'web': self.ShowWebPluginsBanner() elif self.ListPlugins == 'aux': self.ShowAuxPluginsBanner() self.ShowPluginGroupPlugins(self.ListPlugins) def ShowAuxPluginsBanner(self): print(INTRO_BANNER_GENERAL + "\n Available AUXILIARY plugins:" "") def ShowWebPluginsBanner(self): print(INTRO_BANNER_GENERAL + INTRO_BANNER_WEB_PLUGIN_TYPE + "\n Available WEB plugins:" "") def ShowPluginGroupPlugins(self, PluginGroup): for PluginType in self.Core.Config.Plugin.GetTypesForGroup( PluginGroup): self.ShowPluginTypePlugins(PluginType, PluginGroup) def ShowPluginTypePlugins(self, PluginType, PluginGroup): cprint("\n" + '*' * 40 + " " + PluginType.title().replace('_', '-') + " plugins " + '*' * 40) for Plugin in self.Core.Config.Plugin.GetAll(PluginGroup, PluginType): #'Name' : PluginName, 'Code': PluginCode, 'File' : PluginFile, 'Descrip' : PluginDescrip } ) LineStart = " " + Plugin['type'] + ": " + Plugin['name'] Pad1 = "_" * (60 - len(LineStart)) Pad2 = "_" * (20 - len(Plugin['code'])) cprint(LineStart + Pad1 + "(" + Plugin['code'] + ")" + Pad2 + Plugin['descrip'])
class ScannerTests(BaseTestCase): def before(self): self._create_core_mock() self.scanner = Scanner(self.core_mock) def test_ping_sweep_with_full_scan_executes_nmap_and_grep(self): nmap_regex = re.compile("nmap.*[-]PS.*") grep_regex = re.compile("grep.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", grep_regex) self.scanner.ping_sweep("target", "full") def test_ping_sweep_with_arp_scan_executes_nmap_and_grep(self): nmap_regex = re.compile("nmap.*[-]PR.*") grep_regex = re.compile("grep.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", grep_regex) self.scanner.ping_sweep("target", "arp") def test_scan_and_grab_banners_with_tcp_uses_nmap_and_amap_for_fingerprinting(self): nmap_regex = re.compile("nmap.*[-](sV|sS).*[-](sV|sS).*") amap_regex = re.compile("amap.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", amap_regex) self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "tcp", "") def test_scan_and_grab_banners_with_udp_uses_nmap_and_amap_for_fingerprinting(self): nmap_regex = re.compile("nmap.*[-](sV|sU).*[-](sV|sU).*") amap_regex = re.compile("amap.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", amap_regex) self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "udp", "") def test_get_ports_for_service_returns_the_list_of_ports_associated_to_services(self): services = ["snmp", "smb", "smtp", "ms-sql", "ftp", "X11", "ppp", "vnc", "http-rpc-epmap", "msrpc", "http"] flexmock(self.scanner) self.scanner.should_receive("get_nmap_services_file").and_return(NMAP_SERVICES_FILE) for service in services: port_list = self.scanner.get_ports_for_service(service, "") assert_that(isinstance(port_list, list)) assert_that(port_list is not None) def test_target_service_scans_nmap_output_file(self): file_lines = ["Host: 127.0.0.1\tPorts: 7/filtered/tcp//echo//, 80/open/tcp//http/Microsoft IIS\t\n"] flexmock(self.scanner) self.scanner.should_receive("open_file").and_return(FileMock(file_lines)) self.scanner.target_service("nmap_file", "service") def test_probe_service_for_hosts_sets_plugin_list_to_execute_and_returns_http_ports(self): flexmock(self.scanner) self.scanner.should_receive("target_service").and_return("127.0.0.1:80") self.core_mock.Config = flexmock() self.core_mock.Config.should_receive("Set") self.core_mock.PluginHandler = flexmock() self.core_mock.PluginHandler.should_receive("ValidateAndFormatPluginList").once() http_ports = self.scanner.probe_service_for_hosts("nmap_file", "target") assert_that(isinstance(http_ports, list)) assert_that(http_ports, has_length(greater_than(0))) def test_dns_sweep_looks_for_DNS_servers_and_abort_execution_if_no_domain_is_found(self): self._record_dns_sweep_first_steps() flexmock(self.scanner) self._stub_open_file(re.compile(".*\.dns_server.ips"), ["127.0.0.1\n", "127.0.0.1\n"]) self.scanner.should_receive("open_file").with_args(re.compile(".*\.domain_names")).and_raise(IOError).once() self.scanner.dns_sweep("file_with_ips.txt", "file_prefix") def test_dns_sweep_looks_for_DNS_servers_and_tries_to_do_a_zone_transfer_on_found_domains(self): self._record_dns_sweep_first_steps() flexmock(self.scanner) self._stub_open_file(re.compile(".*\.dns_server.ips"), ["127.0.0.1\n", "127.0.0.1\n"]) self._stub_open_file(re.compile(".*\.domain_names"), ["domain1.com"]) self._mock_shell_method_with_args_once("shell_exec", re.compile("host [-]l.*")) # Retrieve domains self._mock_shell_method_with_args_once("shell_exec", re.compile("wc\s[-]l.*cut.*"), return_value=4) # Determines if succeeded self._mock_shell_method_with_args_once("shell_exec", re.compile("rm\s[-]f\s.*\.axfr.*")) self.scanner.dns_sweep("file_with_ips.txt", "file_prefix") def _create_core_mock(self): self.core_mock = flexmock() self.core_mock.Shell = flexmock() self._stub_shell_method("shell_exec", None) def _stub_shell_method(self, method, expected_result): self.core_mock.Shell.should_receive(method).and_return(expected_result) def _mock_shell_method_with_args_once(self, method, args, return_value=None): if (return_value is None): self.core_mock.Shell.should_receive(method).with_args(args).once() else: self.core_mock.Shell.should_receive(method).with_args(args).and_return(return_value).once() def _record_dns_sweep_first_steps(self): nmap_dns_discovery_regex = re.compile("nmap.*[-]sS.*[-]p\s53.*") grep_open_53_port_regex = re.compile("grep.*53/open") rm_old_files_regex = re.compile("rm [-]f .*\.domain_names") self._mock_shell_method_with_args_once("shell_exec", nmap_dns_discovery_regex) self._mock_shell_method_with_args_once("shell_exec", grep_open_53_port_regex) self._mock_shell_method_with_args_once("shell_exec", rm_old_files_regex) def _stub_open_file(self, args, file_lines): returned_file = FileMock(file_lines) self.scanner.should_receive("open_file").with_args(args).and_return(returned_file)
class PluginHandler: PluginCount = 0 def __init__(self, CoreObj, Options): self.Core = CoreObj #This should be dynamic from filesystem: #self.PluginGroups = [ 'web', 'net', 'aux' ] #self.PluginTypes = [ 'passive', 'semi_passive', 'active', 'grep' ] #self.AllowedPluginTypes = self.GetAllowedPluginTypes(Options['PluginType'].split(',')) #self.Simulation, self.Scope, self.PluginGroup, self.Algorithm, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['Algorithm'], Options['ListPlugins'] ] self.Simulation, self.Scope, self.PluginGroup, self.ListPlugins = [ Options['Simulation'], Options['Scope'], Options['PluginGroup'], Options['ListPlugins'] ] self.OnlyPluginsList = self.ValidateAndFormatPluginList(Options['OnlyPlugins']) self.ExceptPluginsList = self.ValidateAndFormatPluginList(Options['ExceptPlugins']) #print "OnlyPlugins="+str(self.OnlyPluginsList) #print "ExceptPlugins="+str(self.ExceptPluginsList) #print "Options['PluginType']="+str(Options['PluginType']) if isinstance(Options['PluginType'], str): # For special plugin types like "quiet" -> "semi_passive" + "passive" Options['PluginType'] = Options['PluginType'].split(',') self.AllowedPlugins = self.Core.DB.Plugin.GetPluginsByGroupType(self.PluginGroup, Options['PluginType']) self.OnlyPluginsSet = len(self.OnlyPluginsList) > 0 self.ExceptPluginsSet = len(self.ExceptPluginsList) > 0 self.scanner = Scanner(self.Core) self.InitExecutionRegistry() self.showOutput = True def ValidateAndFormatPluginList(self, PluginList): List = [] # Ensure there is always a list to iterate from! :) if PluginList != None: List = PluginList ValidatedList = [] #print "List to validate="+str(List) for Item in List: Found = False for Plugin in self.Core.DB.Plugin.GetPluginsByGroup(self.PluginGroup): # Processing Loop if Item in [ Plugin['code'], Plugin['name'] ]: ValidatedList.append(Plugin['code']) Found = True break if not Found: cprint("ERROR: The code '"+Item+"' is not a valid plugin, please use the -l option to see available plugin names and codes") exit() return ValidatedList # Return list of Codes def InitExecutionRegistry(self): # Initialises the Execution registry: As plugins execute they will be tracked here, useful to avoid calling plugins stupidly :) self.ExecutionRegistry = defaultdict(list) for Target in self.Scope: self.ExecutionRegistry[Target] = [] def GetLastPluginExecution(self, Plugin): ExecLog = self.ExecutionRegistry[self.Core.Config.GetTarget()] # Get shorcut to relevant execution log for this target for readability below :) NumItems = len(ExecLog) #print "NumItems="+str(NumItems) if NumItems == 0: return -1 # List is empty #print "NumItems="+str(NumItems) #print str(ExecLog) #print str(range((NumItems -1), 0)) for Index in range((NumItems -1), -1, -1): #print "Index="+str(Index) #print str(ExecLog[Index]) Match = True for Key, Value in ExecLog[Index].items(): # Compare all execution log values against the passed Plugin, if all match, return index to log record if not Key in Plugin or Plugin[Key] != Value: Match = False if Match: #print str(PluginIprint "you have etered " + cnfo)+" was found!" return Index return -1 def PluginAlreadyRun(self, PluginInfo): return self.Core.DB.POutput.PluginAlreadyRun(PluginInfo) def GetExecLogSinceLastExecution(self, Plugin): # Get all execution entries from log since last time the passed plugin executed return self.ExecutionRegistry[self.Core.Config.GetTarget()][self.GetLastPluginExecution(Plugin):] def NormalRequestsAllowed(self): #AllowedPluginTypes = self.Core.Config.GetAllowedPluginTypes('web') #GetAllowedPluginTypes('web') AllowedPluginTypes = self.Core.Config.Plugin.GetAllowedTypes('web') return 'semi_passive' in AllowedPluginTypes or 'active' in AllowedPluginTypes def RequestsPossible(self): # Even passive plugins will make requests to external resources #return [ 'grep' ] != self.Core.Config.GetAllowedPluginTypes('web') return [ 'grep' ] != self.Core.DB.Plugin.GetTypesForGroup('web') def DumpOutputFile(self, Filename, Contents, Plugin, RelativePath=False): SaveDir = self.GetPluginOutputDir(Plugin) abs_path = self.Core.DumpFile(Filename, Contents, SaveDir) if RelativePath: return(os.path.relpath(abs_path, self.Core.Config.GetOutputDirForTargets())) return(abs_path) def RetrieveAbsPath(self, RelativePath): return(os.path.join(self.Core.Config.GetOutputDirForTargets(), RelativePath)) def GetPluginOutputDir(self, Plugin): # Organise results by OWASP Test type and then active, passive, semi_passive #print "Plugin="+str(Plugin)+", Partial url ..="+str(self.Core.Config.Get('partial_url_output_path'))+", TARGET="+self.Core.Config.Get('TARGET') if ((Plugin['group'] == 'web') or (Plugin['group'] == 'net')): return os.path.join(self.Core.DB.Target.GetPath('partial_url_output_path'), WipeBadCharsForFilename(Plugin['title']), Plugin['type']) elif Plugin['group'] == 'aux': return os.path.join(self.Core.Config.Get('AUX_OUTPUT_PATH'), WipeBadCharsForFilename(Plugin['title']), Plugin['type']) def exists(self, directory): return os.path.exists(directory) def GetModule(self, ModuleName, ModuleFile, ModulePath):# Python fiddling to load a module from a file, there is probably a better way... f, Filename, desc = imp.find_module(ModuleFile.split('.')[0], [ModulePath]) #ModulePath = os.path.abspath(ModuleFile) return imp.load_module(ModuleName, f, Filename, desc) def IsChosenPlugin(self, Plugin): Chosen = True if Plugin['group'] == self.PluginGroup: if self.OnlyPluginsSet and Plugin['code'] not in self.OnlyPluginsList: Chosen = False # Skip plugins not present in the white-list defined by the user if self.ExceptPluginsSet and Plugin['code'] in self.ExceptPluginsList: Chosen = False # Skip plugins present in the black-list defined by the user if Plugin['type'] not in self.Core.DB.Plugin.GetTypesForGroup(Plugin['group']): Chosen = False # Skip plugin: Not matching selected type return Chosen def IsActiveTestingPossible(self): # Checks if 1 active plugin is enabled = active testing possible: Possible = False #for PluginType, PluginFile, Title, Code, ReferenceURL in self.Core.Config.GetPlugins(): # Processing Loop #for PluginType, PluginFile, Title, Code in self.Core.Config.Plugin.GetOrder(self.PluginGroup): for Plugin in self.Core.Config.Plugin.GetOrder(self.PluginGroup): if self.IsChosenPlugin(Plugin) and Plugin['type'] == 'active': Possible = True break return Possible def force_overwrite(self): #return self.Core.Config.Get('FORCE_OVERWRITE') return False def CanPluginRun(self, Plugin, ShowMessages = False): #if self.Core.IsTargetUnreachable(): # return False # Cannot run plugin if target is unreachable if not self.IsChosenPlugin(Plugin): return False # Skip not chosen plugins # Grep plugins to be always run and overwritten (they run once after semi_passive and then again after active): #if self.PluginAlreadyRun(Plugin) and not self.Core.Config.Get('FORCE_OVERWRITE'): #not Code == 'OWASP-WU-SPID': # For external plugin forced re-run (development) if self.PluginAlreadyRun(Plugin) and ((not self.force_overwrite() and not ('grep' == Plugin['type'])) or Plugin['type'] == 'external'): #not Code == 'OWASP-WU-SPID': if ShowMessages: logging.info("Plugin: "+Plugin['title']+" ("+Plugin['type']+") has already been run, skipping ..") #if Plugin['Type'] == 'external': # External plugins are run only once per each run, so they are registered for all targets # that are targets in that run. This is an alternative to chaning the js filters etc.. # self.register_plugin_for_all_targets(Plugin) return False if 'grep' == Plugin['type'] and self.PluginAlreadyRun(Plugin): return False # Grep plugins can only run if some active or semi_passive plugin was run since the last time return True def GetPluginFullPath(self, PluginDir, Plugin): return PluginDir+"/"+Plugin['type']+"/"+Plugin['file'] # Path to run the plugin def RunPlugin(self, PluginDir, Plugin, save_output=True): PluginPath = self.GetPluginFullPath(PluginDir, Plugin) (Path, Name) = os.path.split(PluginPath) #(Name, Ext) = os.path.splitext(Name) #self.Core.DB.Debug.Add("Running Plugin -> Plugin="+str(Plugin)+", PluginDir="+str(PluginDir)) PluginOutput = self.GetModule("", Name, Path+"/").run(self.Core, Plugin) #if save_output: #print(PluginOutput) #self.SavePluginInfo(PluginOutput, Plugin) # Timer retrieved here return PluginOutput @staticmethod def rank_plugin(output, pathname): """Rank the current plugin results using PTP. Returns the ranking value. """ def extract_metasploit_modules(cmd): """Extract the metasploit modules contained in the plugin output. Returns the list of (module name, output file) found, an empty list otherwise. """ return [ ( output['output'].get('ModifiedCommand', '').split(' ')[3], os.path.basename( output['output'].get('RelativeFilePath', '')) ) for output in cmd if ('output' in output and 'metasploit' in output['output'].get('ModifiedCommand', ''))] msf_modules = None if output: # Try to retrieve metasploit modules that were used. msf_modules = extract_metasploit_modules(output) owtf_rank = -1 # Default ranking value set to Unknown. try: parser = PTP() if msf_modules: # PTP needs to know the msf module name. for module in msf_modules: parser.parse( pathname=pathname, filename=module[1], # Path to output file. plugin=module[0]) # Metasploit module name. owtf_rank = max( owtf_rank, parser.get_highest_ranking()) else: # Otherwise use the auto-detection mode. parser.parse(pathname=pathname) owtf_rank = parser.get_highest_ranking() except PTPError: # Not supported tool or report not found. pass return owtf_rank def ProcessPlugin(self, plugin_dir, plugin, status={}): # Save how long it takes for the plugin to run. self.Core.Timer.start_timer('Plugin') plugin['start'] = self.Core.Timer.get_start_date_time('Plugin') # Use relative path from targets folders while saving plugin['output_path'] = os.path.relpath( self.GetPluginOutputDir(plugin), self.Core.Config.GetOutputDirForTargets()) status['AllSkipped'] = False # A plugin is going to be run. plugin['status'] = 'Running' self.PluginCount += 1 logging.info( '_' * 10 + ' ' + str(self.PluginCount) + ' - Target: ' + self.Core.DB.Target.GetTargetURL() + ' -> Plugin: ' + plugin['title'] + ' (' + plugin['type'] + ') ' + '_' * 10) # Skip processing in simulation mode, but show until line above # to illustrate what will run if self.Simulation: return None # DB empty => grep plugins will fail, skip!! if ('grep' == plugin['type'] and self.Core.DB.Transaction.NumTransactions() == 0): logging.info( 'Skipped - Cannot run grep plugins: ' 'The Transaction DB is empty') return None output = None status_msg = '' partial_output = [] abort_reason = '' try: output = self.RunPlugin(plugin_dir, plugin) status_msg = 'Successful' status['SomeSuccessful'] = True except KeyboardInterrupt: # Just explain why crashed. status_msg = 'Aborted' abort_reason = 'Aborted by User' status['SomeAborted (Keyboard Interrupt)'] = True except SystemExit: # Abort plugin processing and get out to external exception # handling, information saved elsewhere. raise SystemExit except PluginAbortException as PartialOutput: status_msg = 'Aborted (by user)' partial_output = PartialOutput.parameter abort_reason = 'Aborted by User' status['SomeAborted'] = True except UnreachableTargetException as PartialOutput: status_msg = 'Unreachable Target' partial_output = PartialOutput.parameter abort_reason = 'Unreachable Target' status['SomeAborted'] = True except FrameworkAbortException as PartialOutput: status_msg = 'Aborted (Framework Exit)' partial_output = PartialOutput.parameter abort_reason = 'Framework Aborted' # TODO: Handle this gracefully # except: # Plugin["status"] = "Crashed" # cprint("Crashed") # self.SavePluginInfo(self.Core.Error.Add("Plugin "+Plugin['Type']+"/"+Plugin['File']+" failed for target "+self.Core.Config.Get('TARGET')), Plugin) # Try to save something # TODO: http://blog.tplus1.com/index.php/2007/09/28/the-python-logging-module-is-much-better-than-print-statements/ finally: plugin['status'] = status_msg plugin['end'] = self.Core.Timer.get_end_date_time('Plugin') plugin['owtf_rank'] = self.rank_plugin( output, self.GetPluginOutputDir(plugin)) if status_msg == 'Successful': self.Core.DB.POutput.SavePluginOutput(plugin, output) else: self.Core.DB.POutput.SavePartialPluginOutput( plugin, partial_output, abort_reason) if status_msg == 'Aborted': self.Core.Error.UserAbort('Plugin') if abort_reason == 'Framework Aborted': self.Core.Finish('Aborted') return output def ProcessPlugins(self): status = { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True} if self.PluginGroup in ['web', 'aux', 'net']: self.ProcessPluginsForTargetList( self.PluginGroup, status, self.Core.DB.Target.GetAll("id")) return status def GetPluginGroupDir(self, PluginGroup): PluginDir = self.Core.Config.FrameworkConfigGet('PLUGINS_DIR')+PluginGroup return PluginDir def SwitchToTarget(self, Target): self.Core.DB.Target.SetTarget(Target) # Tell Target DB that all Gets/Sets are now Target-specific def get_plugins_in_order_for_PluginGroup(self, PluginGroup): return self.Core.Config.Plugin.GetOrder(PluginGroup) def get_plugins_in_order(self, PluginGroup): return self.Core.Config.Plugin.GetOrder(PluginGroup) def ProcessPluginsForTargetList(self, PluginGroup, Status, TargetList): # TargetList param will be useful for netsec stuff to call this PluginDir = self.GetPluginGroupDir(PluginGroup) if PluginGroup == 'net': portwaves = self.Core.Config.Get('PORTWAVES') waves = portwaves.split(',') waves.append('-1') lastwave=0 for Target in TargetList: # For each Target self.scanner.scan_network(Target) #Scanning and processing the first part of the ports for i in range(1): ports = self.Core.Config.GetTcpPorts(lastwave,waves[i]) print "probing for ports" + str(ports) http = self.scanner.probe_network(Target, 'tcp', ports) # Tell Config that all Gets/Sets are now # Target-specific. self.SwitchToTarget(Target) for Plugin in self.get_plugins_in_order_for_PluginGroup(PluginGroup): self.ProcessPlugin(PluginDir, Plugin, Status) lastwave = waves[i] for http_ports in http: if http_ports == '443': self.ProcessPluginsForTargetList( 'web', { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True}, {'https://' + Target.split('//')[1]} ) else: self.ProcessPluginsForTargetList( 'web', { 'SomeAborted': False, 'SomeSuccessful': False, 'AllSkipped': True}, {Target} ) else: pass #self.WorkerManager.startinput() #self.WorkerManager.fillWorkList(PluginGroup,TargetList) #self.WorkerManager.spawn_workers() #self.WorkerManager.manage_workers() #self.WorkerManager.poisonPillToWorkers() #Status = self.WorkerManager.joinWorker() #if 'breadth' == self.Algorithm: # Loop plugins, then targets # for Plugin in self.Core.Config.Plugin.GetOrder(PluginGroup):# For each Plugin # #print "Processing Plugin="+str(Plugin) # for Target in TargetList: # For each Target # #print "Processing Target="+str(Target) # self.SwitchToTarget(Target) # Tell Config that all Gets/Sets are now Target-specific # self.ProcessPlugin( PluginDir, Plugin, Status ) #elif 'depth' == self.Algorithm: # Loop Targets, then plugins # for Target in TargetList: # For each Target # self.SwitchToTarget(Target) # Tell Config that all Gets/Sets are now Target-specific # for Plugin in self.Core.Config.Plugin.GetOrder(PluginGroup):# For each Plugin # self.ProcessPlugin( PluginDir, Plugin, Status ) def CleanUp(self): self.WorkerManager.clean_up() def SavePluginInfo(self, PluginOutput, Plugin): self.Core.DB.SaveDBs() # Save new URLs to DB after each request self.Core.Reporter.SavePluginReport(PluginOutput, Plugin) # Timer retrieved by Reporter def ShowPluginList(self): if self.ListPlugins == 'web': self.ShowWebPluginsBanner() elif self.ListPlugins == 'aux': self.ShowAuxPluginsBanner() self.ShowPluginGroupPlugins(self.ListPlugins) def ShowAuxPluginsBanner(self): print(INTRO_BANNER_GENERAL+"\n Available AUXILIARY plugins:""") def ShowWebPluginsBanner(self): print(INTRO_BANNER_GENERAL+INTRO_BANNER_WEB_PLUGIN_TYPE+"\n Available WEB plugins:""") def ShowPluginGroupPlugins(self, PluginGroup): for PluginType in self.Core.Config.Plugin.GetTypesForGroup(PluginGroup): self.ShowPluginTypePlugins(PluginType,PluginGroup) def ShowPluginTypePlugins(self, PluginType,PluginGroup): cprint("\n"+'*' * 40+" "+PluginType.title().replace('_', '-')+" plugins "+'*' * 40) for Plugin in self.Core.Config.Plugin.GetAll(PluginGroup, PluginType): #'Name' : PluginName, 'Code': PluginCode, 'File' : PluginFile, 'Descrip' : PluginDescrip } ) LineStart = " "+Plugin['type']+": "+Plugin['name'] Pad1 = "_" * (60 - len(LineStart)) Pad2 = "_" * (20- len(Plugin['code'])) cprint(LineStart+Pad1+"("+Plugin['code']+")"+Pad2+Plugin['descrip'])
class ScannerTests(BaseTestCase): def before(self): self._create_core_mock() self.scanner = Scanner(self.core_mock) def test_ping_sweep_with_full_scan_executes_nmap_and_grep(self): nmap_regex = re.compile("nmap.*[-]PS.*") grep_regex = re.compile("grep.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", grep_regex) self.scanner.ping_sweep("target", "full") def test_ping_sweep_with_arp_scan_executes_nmap_and_grep(self): nmap_regex = re.compile("nmap.*[-]PR.*") grep_regex = re.compile("grep.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", grep_regex) self.scanner.ping_sweep("target", "arp") def test_scan_and_grab_banners_with_tcp_uses_nmap_and_amap_for_fingerprinting(self): nmap_regex = re.compile("nmap.*[-](sV|sS).*[-](sV|sS).*") amap_regex = re.compile("amap.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", amap_regex) self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "tcp", "") def test_scan_and_grab_banners_with_udp_uses_nmap_and_amap_for_fingerprinting(self): nmap_regex = re.compile("nmap.*[-](sV|sU).*[-](sV|sU).*") amap_regex = re.compile("amap.*") self._mock_shell_method_with_args_once("shell_exec", nmap_regex) self._mock_shell_method_with_args_once("shell_exec", amap_regex) self.scanner.scan_and_grab_banners("file_with_ips", "file_prefix", "udp", "") def test_get_ports_for_service_returns_the_list_of_ports_associated_to_services(self): services = ["snmp","smb","smtp","ms-sql","ftp","X11","ppp","vnc","http-rpc-epmap","msrpc","http"] flexmock(self.scanner) self.scanner.should_receive("get_nmap_services_file").and_return(NMAP_SERVICES_FILE) for service in services: port_list = self.scanner.get_ports_for_service(service, "") assert_that(isinstance(port_list, list)) assert_that(port_list is not None) def test_target_service_scans_nmap_output_file(self): file_content = "Host: 127.0.0.1\tPorts: 7/filtered/tcp//echo//, 80/open/tcp//http/Microsoft IIS\t\n" fake_file = flexmock() fake_file.should_receive("read").and_return(file_content).once() fake_file.should_receive("close").once() flexmock(self.scanner) self.scanner.should_receive("get_ports_for_service").and_return(["7", "80"]) self.scanner.should_receive("open_file").and_return(fake_file) self.scanner.target_service("nmap_file", "service") def test_probe_service_for_hosts_sets_plugin_list_to_execute_and_returns_http_ports(self): flexmock(self.scanner) self.scanner.should_receive("target_service").and_return("127.0.0.1:80") self.core_mock.Config = flexmock() self.core_mock.Config.should_receive("Set") self.core_mock.PluginHandler = flexmock() self.core_mock.PluginHandler.should_receive("ValidateAndFormatPluginList").once() http_ports = self.scanner.probe_service_for_hosts("nmap_file", "target") assert_that(isinstance(http_ports, list)) assert_that(http_ports, has_length(greater_than(0))) def _create_core_mock(self): self.core_mock = flexmock() self.core_mock.Shell = flexmock() self._stub_shell_method("shell_exec", None) def _stub_shell_method(self, method, expected_result): self.core_mock.Shell.should_receive(method).and_return(expected_result) def _mock_shell_method_with_args_once(self, method, args): self.core_mock.Shell.should_receive(method).with_args(args).once()