def test_webinspect_api_helper_get_scan_status_success(api_mock, json_loads_mock): # Given webinspect_api_helper_object = WebInspectAPIHelper(silent=True, webinspect_setting_overrides=MagicMock()) webinspect_api_helper_object.api.get_current_status = api_mock json_loads_mock.side_effect = None # When webinspect_api_helper_object.get_scan_status("test_guid") # Expect assert api_mock.call_count == 1
def test_webinspect_api_helper_get_scan_status_failure_unbound_local_error(api_mock, json_loads_mock, log_error_mock): # Given webinspect_api_helper_object = WebInspectAPIHelper(silent=True, webinspect_setting_overrides=MagicMock()) webinspect_api_helper_object.api.get_current_status = api_mock json_loads_mock.side_effect = UnboundLocalError # When webinspect_api_helper_object.get_scan_status("test_guid") # Expect assert log_error_mock.call_count == 1 assert api_mock.call_count == 1
def download(server, scan_name, scan_id, extension, username, password): try: auth_config = WebInspectAuth() username, password = auth_config.authenticate(username, password) query_client = WebInspectAPIHelper(host=server, username=username, password=password) if not scan_id: results = query_client.get_scan_by_name(scan_name) if len(results) == 0: webinspect_logexceptionhelper.log_error_no_scans_found(scan_name) elif len(results) == 1: scan_id = results[0]['ID'] Logger.app.info("Scan matching the name {} found.".format(scan_name)) Logger.app.info("Downloading scan {}".format(scan_name)) query_client.export_scan_results(scan_id, extension, scan_name) else: webinspect_logexceptionhelper.log_info_multiple_scans_found(scan_name) print("{0:80} {1:40} {2:10}".format('Scan Name', 'Scan ID', 'Scan Status')) print("{0:80} {1:40} {2:10}\n".format('-' * 80, '-' * 40, '-' * 10)) for result in results: print("{0:80} {1:40} {2:10}".format(result['Name'], result['ID'], result['Status'])) else: if query_client.get_scan_status(scan_id): query_client.export_scan_results(scan_id, extension, scan_name) else: if query_client.get_scan_status(scan_id): query_client.export_scan_results(scan_id, extension, scan_name) else: Logger.console.error("Unable to find scan with ID matching {}".format(scan_id)) except (UnboundLocalError, TypeError) as e: webinspect_logexceptionhelper.log_error_webinspect_download(e) # If we've made it this far, our new credentials are valid and should be saved if username is not None and password is not None and not auth_config.has_auth_creds(): auth_config.write_credentials(username, password)
class WebInspectScan: def __init__(self, cli_overrides): # used for multi threading the _is_available API call self.config = WebInspectConfig() # handle all the overrides if 'git' not in cli_overrides: # it shouldn't be in the overrides, but here for potential future support of cli passed git paths cli_overrides['git'] = Config().git self.scan_overrides = ScanOverrides(cli_overrides) # run the scan self.scan() def scan(self): """ Start a scan for webinspect. It is multithreaded in that it uses a thread to handle checking on the scan status and a queue in the main execution to wait for a repsonse from the thread. :return: """ # handle the authentication auth_config = WebInspectAuth() username, password = auth_config.authenticate(self.scan_overrides.username, self.scan_overrides.password) # handle github setup self._webinspect_git_clone() self._set_api(username=username, password=password) # self.webinspect_api = WebInspectAPIHelper(username=username, password=password, # webinspect_setting_overrides=self.scan_overrides) # abstract out a bunch of conditional uploads self._upload_settings_and_policies() try: Logger.app.debug("Running WebInspect Scan") self.scan_id = self.webinspect_api.create_scan() # context manager to handle interrupts properly with self._termination_event_handler(): self._scan() except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: webinspectloghelper.log_error_scan_start_failed(e) exit(ExitStatus.failure) Logger.app.info("WebInspect Scan Complete.") # If we've made it this far, our new credentials are valid and should be saved if username is not None and password is not None and not auth_config.has_auth_creds(): auth_config.write_credentials(username, password) def _set_api(self, username, password): """ created so I could mock this functionality better. It sets up the webinspect api :param username: :param password: :return: """ self.webinspect_api = WebInspectAPIHelper(username=username, password=password, webinspect_setting_overrides=self.scan_overrides) def _upload_settings_and_policies(self): """ upload any settings, policies or macros that need to be uploaded :return: """ # if a scan policy has been specified, we need to make sure we can find/use it on the server self.webinspect_api.verify_scan_policy(self.config) # Upload whatever overrides have been provided, skipped unless explicitly declared if self.webinspect_api.setting_overrides.webinspect_upload_settings: self.webinspect_api.upload_settings() if self.webinspect_api.setting_overrides.webinspect_upload_webmacros: self.webinspect_api.upload_webmacros() # if there was a provided scan policy, we've already uploaded so don't bother doing it again. if self.webinspect_api.setting_overrides.webinspect_upload_policy and not self.webinspect_api.setting_overrides.scan_policy: self.webinspect_api.upload_policy() def _scan(self, delay=2): """ If it returns complete we are happy and download the results files. If we enter NotRunning then something has gone wrong and we want to exit with a failure. :param scan_id: the id on the webinspect server for the running scan :param delay: time between calls to Webinspect server :return: no return but upon completion sends a "complete" message back to the queue that is waiting for it. """ # self.webinspect_server = self.webinspect_api.setting_overrides.endpoint self.webinspect_api.host = self.webinspect_api.setting_overrides.endpoint scan_complete = False while not scan_complete: current_status = self.webinspect_api.get_scan_status(self.scan_id) if current_status.lower() == 'complete': # Now let's download or export the scan artifact in two formats self.webinspect_api.export_scan_results(self.scan_id, 'fpr') self.webinspect_api.export_scan_results(self.scan_id, 'xml') return # TODO add json export elif current_status.lower() == 'notrunning': webinspectloghelper.log_error_not_running_scan() self._stop_scan(self.scan_id) sys.exit(ExitStatus.failure) time.sleep(delay) def _stop_scan(self, scan_id): self.webinspect_api.stop_scan(scan_id) # below functions are for handling someone forcefully ending webbreaker. def _exit_scan_gracefully(self, *args): """ called when someone ctl+c's - sends an api call to end the running scan. :param args: :return: """ Logger.app.info("Aborting!") self.webinspect_api.stop_scan(self.scan_id) exit(ExitStatus.failure) @contextmanager def _termination_event_handler(self): """ meant to handle termination events (ctr+c and more) so that we call scan_end(scan_id) if a user decides to end the scan. :return: """ # Intercept the "please terminate" signals original_sigint_handler = getsignal(SIGINT) original_sigabrt_handler = getsignal(SIGABRT) original_sigterm_handler = getsignal(SIGTERM) for sig in (SIGABRT, SIGINT, SIGTERM): signal(sig, self._exit_scan_gracefully) yield # needed for context manager # Go back to normal signal handling signal(SIGABRT, original_sigabrt_handler) signal(SIGINT, original_sigint_handler) signal(SIGTERM, original_sigterm_handler) def _webinspect_git_clone(self): """ If local file exist, it will use that file. If not, it will go to github and clone the config files :return: """ try: config_helper = Config() etc_dir = config_helper.etc git_dir = os.path.join(config_helper.git, '.git') try: if self.scan_overrides.settings == 'Default': webinspectloghelper.log_info_default_settings() if os.path.isfile(self.scan_overrides.webinspect_upload_settings + '.xml'): self.scan_overrides.webinspect_upload_settings = self.scan_overrides.webinspect_upload_settings + '.xml' elif os.path.exists(git_dir): webinspectloghelper.log_info_updating_webinspect_configurations(etc_dir) check_output(['git', 'init', etc_dir]) check_output( ['git', '--git-dir=' + git_dir, '--work-tree=' + str(config_helper.git), 'reset', '--hard']) check_output( ['git', '--git-dir=' + git_dir, '--work-tree=' + str(config_helper.git), 'pull', '--rebase']) sys.stdout.flush() elif not os.path.exists(git_dir): webinspectloghelper.log_info_webinspect_git_clonning(config_helper.git) check_output(['git', 'clone', self.config.webinspect_git, config_helper.git]) else: Logger.app.error( "No GIT Repo was declared in your config.ini, therefore nothing will be cloned!") except (CalledProcessError, AttributeError) as e: webinspectloghelper.log_webinspect_config_issue(e) raise except GitCommandError as e: webinspectloghelper.log_git_access_error(self.config.webinspect_git, e) exit(ExitStatus.failure) except IndexError as e: webinspectloghelper.log_config_file_unavailable(e) exit(ExitStatus.failure) Logger.app.debug("Completed webinspect config fetch") except TypeError as e: webinspectloghelper.log_error_git_cloning_error(e)
class WebInspectScan: def __init__(self, cli_overrides): # keep track on when the scan starts self.start_time = self._get_time() # used for multi threading the _is_available API call self.config = WebInspectConfig() # handle all the overrides if 'git' not in cli_overrides: # it shouldn't be in the overrides, but here for potential future support of cli passed git paths cli_overrides['git'] = Config().git self._webinspect_git_clone(cli_overrides['settings']) self.scan_overrides = ScanOverrides(cli_overrides) # run the scan self.scan() def scan(self): """ Start a scan for webinspect. It is multithreaded in that it uses a thread to handle checking on the scan status and a queue in the main execution to wait for a repsonse from the thread. :return: """ # handle the authentication auth_config = WebInspectAuth() username, password = auth_config.authenticate( self.scan_overrides.username, self.scan_overrides.password) self._set_api(username=username, password=password) # abstract out a bunch of conditional uploads self._upload_settings_and_policies() try: Logger.app.debug("Running WebInspect Scan") self.scan_id = self.webinspect_api.create_scan() # context manager to handle interrupts properly #with self._termination_event_handler(): self._scan() except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e: webinspectloghelper.log_error_scan_start_failed(e) sys.exit(ExitStatus.failure) Logger.app.debug("WebInspect Scan Complete.") # If we've made it this far, our new credentials are valid and should be saved if username is not None and password is not None and not auth_config.has_auth_creds( ): auth_config.write_credentials(username, password) #parse through xml file after scan try: file_name = self.scan_overrides.scan_name + '.xml' self.xml_parsing(file_name) except IOError as e: webinspectloghelper.log_error_failed_scan_export(e) def xml_parsing(self, file_name): """ if scan complete, open and parse through the xml file and output <host>, <severity>, <vulnerability>, <CWE> in console :return: JSON file """ tree = ET.ElementTree(file=file_name) root = tree.getroot() vulnerabilities = Vulnerabilities() for elem in root.findall('Session'): vulnerability = Vulnerability() vulnerability.payload_url = elem.find('URL').text # This line should be: for issue in elem.iter(tag='Issue'): # But because of a bug in python 2 it has to be this way. # https://stackoverflow.com/questions/29695794/typeerror-iter-takes-no-keyword-arguments for issue in elem.iter('Issue'): vulnerability.vulnerability_name = issue.find('Name').text vulnerability.severity = issue.find('Severity').text vulnerability.webinspect_id = issue.attrib vulnerability.cwe = [] for cwe in issue.iter('Classification'): vulnerability.cwe.append(cwe.text) vulnerabilities.add(vulnerability) Logger.app.info("Exporting scan: {0} as {1}".format( self.scan_id, 'json')) Logger.app.info("Scan results file is available: {0}{1}".format( self.scan_overrides.scan_name, '.json')) # keep track on when the scan ends end_time = self._get_time() vulnerabilities.write_to_console(self.scan_overrides.scan_name) vulnerabilities.write_to_json(file_name, self.scan_overrides.scan_name, self.scan_id, self.start_time, end_time) Logger.app.info("Scan start time: {}".format(self.start_time)) Logger.app.info("Scan end time: {}".format(end_time)) def _get_time(self): return datetime.utcfromtimestamp( time.time()).strftime('%Y-%m-%d %H:%M:%S') def _set_api(self, username, password): """ created so I could mock this functionality better. It sets up the webinspect api :param username: :param password: :return: """ self.webinspect_api = WebInspectAPIHelper( username=username, password=password, webinspect_setting_overrides=self.scan_overrides) def _upload_settings_and_policies(self): """ upload any settings, policies or macros that need to be uploaded :return: """ # if a scan policy has been specified, we need to make sure we can find/use it on the server self.webinspect_api.verify_scan_policy(self.config) # Upload whatever overrides have been provided, skipped unless explicitly declared if self.webinspect_api.setting_overrides.webinspect_upload_settings: self.webinspect_api.upload_settings() if self.webinspect_api.setting_overrides.webinspect_upload_webmacros: self.webinspect_api.upload_webmacros() # if there was a provided scan policy, we've already uploaded so don't bother doing it again. if self.webinspect_api.setting_overrides.webinspect_upload_policy and not self.webinspect_api.setting_overrides.scan_policy: self.webinspect_api.upload_policy() #def _scan(self, delay=10): @CircuitBreaker(fail_max=5, reset_timeout=60) def _scan(self): """ If it returns complete we are happy and download the results files. If we enter NotRunning then something has gone wrong and we want to exit with a failure. :param scan_id: the id on the webinspect server for the running scan :param delay: time between calls to Webinspect server :return: no return but upon completion sends a "complete" message back to the queue that is waiting for it. """ # self.webinspect_server = self.webinspect_api.setting_overrides.endpoint self.webinspect_api.host = self.webinspect_api.setting_overrides.endpoint scan_complete = False while not scan_complete: current_status = self.webinspect_api.get_scan_status(self.scan_id) try: # Happy path - we completed our scan if current_status.lower() == 'complete': # Now let's download or export the scan artifact in two formats self.webinspect_api.export_scan_results( self.scan_id, 'fpr') self.webinspect_api.export_scan_results( self.scan_id, 'xml') return # TODO add json export # The scan can sometimes go from running to not running and that is not what we want. elif current_status.lower() == 'notrunning': webinspectloghelper.log_error_not_running_scan() self._stop_scan(self.scan_id) sys.exit(ExitStatus.failure) # This is interesting behavior and we want to log it. # It should never be in a state besides Running, NotRunning and Complete. elif current_status.lower() != "running": webinspectloghelper.log_error_scan_in_weird_state( scan_name=self.scan_id, state=current_status) sys.exit(ExitStatus.failure) #time.sleep(delay) # Sometimes we are not able to get current_status and it is a None response. except AttributeError as e: webinspectloghelper.log_error_unrecoverable_scan( current_status, e) sys.exit(ExitStatus.failure) def _stop_scan(self, scan_id): self.webinspect_api.stop_scan(scan_id) # below functions are for handling someone forcefully ending webbreaker. def _exit_scan_gracefully(self, *args): """ called when someone ctl+c's - sends an api call to end the running scan. :param args: :return: """ Logger.app.info("Aborting!") self.webinspect_api.stop_scan(self.scan_id) sys.exit(ExitStatus.failure) @contextmanager def _termination_event_handler(self): """ meant to handle termination events (ctr+c and more) so that we call scan_end(scan_id) if a user decides to end the scan. :return: """ # Intercept the "please terminate" signals original_sigint_handler = getsignal(SIGINT) original_sigabrt_handler = getsignal(SIGABRT) original_sigterm_handler = getsignal(SIGTERM) for sig in (SIGABRT, SIGINT, SIGTERM): signal(sig, self._exit_scan_gracefully) yield # needed for context manager # Go back to normal signal handling signal(SIGABRT, original_sigabrt_handler) signal(SIGINT, original_sigint_handler) signal(SIGTERM, original_sigterm_handler) def _webinspect_git_clone(self, cli_settings): """ If local file exist, it will use that file. If not, it will go to github and clone the config files :return: """ try: config_helper = Config() etc_dir = config_helper.etc git_dir = os.path.join(config_helper.git, '.git') try: if cli_settings == 'Default': webinspectloghelper.log_info_default_settings() elif os.path.exists(git_dir): webinspectloghelper.log_info_updating_webinspect_configurations( etc_dir) check_output(['git', 'init', etc_dir]) check_output([ 'git', '--git-dir=' + git_dir, '--work-tree=' + str(config_helper.git), 'reset', '--hard' ]) check_output([ 'git', '--git-dir=' + git_dir, '--work-tree=' + str(config_helper.git), 'pull', '--rebase' ]) sys.stdout.flush() elif not os.path.exists(git_dir): webinspectloghelper.log_info_webinspect_git_clonning( config_helper.git) check_output([ 'git', 'clone', self.config.webinspect_git, config_helper.git ]) else: Logger.app.error( "No GIT Repo was declared in your config.ini, therefore nothing will be cloned!" ) except (CalledProcessError, AttributeError) as e: webinspectloghelper.log_webinspect_config_issue(e) raise except GitCommandError as e: webinspectloghelper.log_git_access_error( self.config.webinspect_git, e) sys.exit(ExitStatus.failure) except IndexError as e: webinspectloghelper.log_config_file_unavailable(e) sys.exit(ExitStatus.failure) Logger.app.debug("Completed webinspect config fetch") except TypeError as e: webinspectloghelper.log_error_git_cloning_error(e)