def test_webinspect_api_helper_upload_settings_success(api_mock):
# Given
webinspect_api_helper_object = WebInspectAPIHelper(silent=True, webinspect_setting_overrides=MagicMock())
webinspect_api_helper_object.api.upload_settings = api_mock
# When
webinspect_api_helper_object.upload_settings()
# Expect
assert api_mock.call_count == 1
def test_webinspect_api_helper_upload_settings_failed_name_error(api_mock, log_error_mock, log_no_server_mock):
# Given
webinspect_api_helper_object = WebInspectAPIHelper(silent=True, webinspect_setting_overrides=MagicMock())
api_mock.side_effect = NameError
webinspect_api_helper_object.api.upload_settings = api_mock
# When
webinspect_api_helper_object.upload_settings()
# Expect
assert log_no_server_mock.call_count == 1
assert log_error_mock.call_count == 1
assert api_mock.call_count == 1
class WebInspectScan:
def __init__(self, cli_overrides):
# used for multi threading the _is_available API call
self.config = WebInspectConfig()
# handle all the overrides
if 'git' not in cli_overrides: # it shouldn't be in the overrides, but here for potential future support of cli passed git paths
cli_overrides['git'] = Config().git
self.scan_overrides = ScanOverrides(cli_overrides)
# run the scan
self.scan()
def scan(self):
"""
Start a scan for webinspect. It is multithreaded in that it uses a thread to handle checking on the scan status
and a queue in the main execution to wait for a repsonse from the thread.
:return:
"""
# handle the authentication
auth_config = WebInspectAuth()
username, password = auth_config.authenticate(self.scan_overrides.username, self.scan_overrides.password)
# handle github setup
self._webinspect_git_clone()
self._set_api(username=username, password=password)
# self.webinspect_api = WebInspectAPIHelper(username=username, password=password,
# webinspect_setting_overrides=self.scan_overrides)
# abstract out a bunch of conditional uploads
self._upload_settings_and_policies()
try:
Logger.app.debug("Running WebInspect Scan")
self.scan_id = self.webinspect_api.create_scan()
# context manager to handle interrupts properly
with self._termination_event_handler():
self._scan()
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
webinspectloghelper.log_error_scan_start_failed(e)
exit(ExitStatus.failure)
Logger.app.info("WebInspect Scan Complete.")
# If we've made it this far, our new credentials are valid and should be saved
if username is not None and password is not None and not auth_config.has_auth_creds():
auth_config.write_credentials(username, password)
def _set_api(self, username, password):
"""
created so I could mock this functionality better. It sets up the webinspect api
:param username:
:param password:
:return:
"""
self.webinspect_api = WebInspectAPIHelper(username=username, password=password,
webinspect_setting_overrides=self.scan_overrides)
def _upload_settings_and_policies(self):
"""
upload any settings, policies or macros that need to be uploaded
:return:
"""
# if a scan policy has been specified, we need to make sure we can find/use it on the server
self.webinspect_api.verify_scan_policy(self.config)
# Upload whatever overrides have been provided, skipped unless explicitly declared
if self.webinspect_api.setting_overrides.webinspect_upload_settings:
self.webinspect_api.upload_settings()
if self.webinspect_api.setting_overrides.webinspect_upload_webmacros:
self.webinspect_api.upload_webmacros()
# if there was a provided scan policy, we've already uploaded so don't bother doing it again.
if self.webinspect_api.setting_overrides.webinspect_upload_policy and not self.webinspect_api.setting_overrides.scan_policy:
self.webinspect_api.upload_policy()
def _scan(self, delay=2):
"""
If it returns complete we are
happy and download the results files. If we enter NotRunning then something has gone wrong and we want to
exit with a failure.
:param scan_id: the id on the webinspect server for the running scan
:param delay: time between calls to Webinspect server
:return: no return but upon completion sends a "complete" message back to the queue that is waiting for it.
"""
# self.webinspect_server = self.webinspect_api.setting_overrides.endpoint
self.webinspect_api.host = self.webinspect_api.setting_overrides.endpoint
scan_complete = False
while not scan_complete:
current_status = self.webinspect_api.get_scan_status(self.scan_id)
if current_status.lower() == 'complete':
# Now let's download or export the scan artifact in two formats
self.webinspect_api.export_scan_results(self.scan_id, 'fpr')
self.webinspect_api.export_scan_results(self.scan_id, 'xml')
return
# TODO add json export
elif current_status.lower() == 'notrunning':
webinspectloghelper.log_error_not_running_scan()
self._stop_scan(self.scan_id)
sys.exit(ExitStatus.failure)
time.sleep(delay)
def _stop_scan(self, scan_id):
self.webinspect_api.stop_scan(scan_id)
# below functions are for handling someone forcefully ending webbreaker.
def _exit_scan_gracefully(self, *args):
"""
called when someone ctl+c's - sends an api call to end the running scan.
:param args:
:return:
"""
Logger.app.info("Aborting!")
self.webinspect_api.stop_scan(self.scan_id)
exit(ExitStatus.failure)
@contextmanager
def _termination_event_handler(self):
"""
meant to handle termination events (ctr+c and more) so that we call scan_end(scan_id) if a user decides to end the
scan.
:return:
"""
# Intercept the "please terminate" signals
original_sigint_handler = getsignal(SIGINT)
original_sigabrt_handler = getsignal(SIGABRT)
original_sigterm_handler = getsignal(SIGTERM)
for sig in (SIGABRT, SIGINT, SIGTERM):
signal(sig, self._exit_scan_gracefully)
yield # needed for context manager
# Go back to normal signal handling
signal(SIGABRT, original_sigabrt_handler)
signal(SIGINT, original_sigint_handler)
signal(SIGTERM, original_sigterm_handler)
def _webinspect_git_clone(self):
"""
If local file exist, it will use that file. If not, it will go to github and clone the config files
:return:
"""
try:
config_helper = Config()
etc_dir = config_helper.etc
git_dir = os.path.join(config_helper.git, '.git')
try:
if self.scan_overrides.settings == 'Default':
webinspectloghelper.log_info_default_settings()
if os.path.isfile(self.scan_overrides.webinspect_upload_settings + '.xml'):
self.scan_overrides.webinspect_upload_settings = self.scan_overrides.webinspect_upload_settings + '.xml'
elif os.path.exists(git_dir):
webinspectloghelper.log_info_updating_webinspect_configurations(etc_dir)
check_output(['git', 'init', etc_dir])
check_output(
['git', '--git-dir=' + git_dir, '--work-tree=' + str(config_helper.git), 'reset', '--hard'])
check_output(
['git', '--git-dir=' + git_dir, '--work-tree=' + str(config_helper.git), 'pull', '--rebase'])
sys.stdout.flush()
elif not os.path.exists(git_dir):
webinspectloghelper.log_info_webinspect_git_clonning(config_helper.git)
check_output(['git', 'clone', self.config.webinspect_git, config_helper.git])
else:
Logger.app.error(
"No GIT Repo was declared in your config.ini, therefore nothing will be cloned!")
except (CalledProcessError, AttributeError) as e:
webinspectloghelper.log_webinspect_config_issue(e)
raise
except GitCommandError as e:
webinspectloghelper.log_git_access_error(self.config.webinspect_git, e)
exit(ExitStatus.failure)
except IndexError as e:
webinspectloghelper.log_config_file_unavailable(e)
exit(ExitStatus.failure)
Logger.app.debug("Completed webinspect config fetch")
except TypeError as e:
webinspectloghelper.log_error_git_cloning_error(e)
class WebInspectScan:
def __init__(self, cli_overrides):
# keep track on when the scan starts
self.start_time = self._get_time()
# used for multi threading the _is_available API call
self.config = WebInspectConfig()
# handle all the overrides
if 'git' not in cli_overrides: # it shouldn't be in the overrides, but here for potential future support of cli passed git paths
cli_overrides['git'] = Config().git
self._webinspect_git_clone(cli_overrides['settings'])
self.scan_overrides = ScanOverrides(cli_overrides)
# run the scan
self.scan()
def scan(self):
"""
Start a scan for webinspect. It is multithreaded in that it uses a thread to handle checking on the scan status
and a queue in the main execution to wait for a repsonse from the thread.
:return:
"""
# handle the authentication
auth_config = WebInspectAuth()
username, password = auth_config.authenticate(
self.scan_overrides.username, self.scan_overrides.password)
self._set_api(username=username, password=password)
# abstract out a bunch of conditional uploads
self._upload_settings_and_policies()
try:
Logger.app.debug("Running WebInspect Scan")
self.scan_id = self.webinspect_api.create_scan()
# context manager to handle interrupts properly
#with self._termination_event_handler():
self._scan()
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError) as e:
webinspectloghelper.log_error_scan_start_failed(e)
sys.exit(ExitStatus.failure)
Logger.app.debug("WebInspect Scan Complete.")
# If we've made it this far, our new credentials are valid and should be saved
if username is not None and password is not None and not auth_config.has_auth_creds(
):
auth_config.write_credentials(username, password)
#parse through xml file after scan
try:
file_name = self.scan_overrides.scan_name + '.xml'
self.xml_parsing(file_name)
except IOError as e:
webinspectloghelper.log_error_failed_scan_export(e)
def xml_parsing(self, file_name):
"""
if scan complete, open and parse through the xml file and output <host>, <severity>, <vulnerability>, <CWE> in console
:return: JSON file
"""
tree = ET.ElementTree(file=file_name)
root = tree.getroot()
vulnerabilities = Vulnerabilities()
for elem in root.findall('Session'):
vulnerability = Vulnerability()
vulnerability.payload_url = elem.find('URL').text
# This line should be: for issue in elem.iter(tag='Issue'):
# But because of a bug in python 2 it has to be this way.
# https://stackoverflow.com/questions/29695794/typeerror-iter-takes-no-keyword-arguments
for issue in elem.iter('Issue'):
vulnerability.vulnerability_name = issue.find('Name').text
vulnerability.severity = issue.find('Severity').text
vulnerability.webinspect_id = issue.attrib
vulnerability.cwe = []
for cwe in issue.iter('Classification'):
vulnerability.cwe.append(cwe.text)
vulnerabilities.add(vulnerability)
Logger.app.info("Exporting scan: {0} as {1}".format(
self.scan_id, 'json'))
Logger.app.info("Scan results file is available: {0}{1}".format(
self.scan_overrides.scan_name, '.json'))
# keep track on when the scan ends
end_time = self._get_time()
vulnerabilities.write_to_console(self.scan_overrides.scan_name)
vulnerabilities.write_to_json(file_name, self.scan_overrides.scan_name,
self.scan_id, self.start_time, end_time)
Logger.app.info("Scan start time: {}".format(self.start_time))
Logger.app.info("Scan end time: {}".format(end_time))
def _get_time(self):
return datetime.utcfromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S')
def _set_api(self, username, password):
"""
created so I could mock this functionality better. It sets up the webinspect api
:param username:
:param password:
:return:
"""
self.webinspect_api = WebInspectAPIHelper(
username=username,
password=password,
webinspect_setting_overrides=self.scan_overrides)
def _upload_settings_and_policies(self):
"""
upload any settings, policies or macros that need to be uploaded
:return:
"""
# if a scan policy has been specified, we need to make sure we can find/use it on the server
self.webinspect_api.verify_scan_policy(self.config)
# Upload whatever overrides have been provided, skipped unless explicitly declared
if self.webinspect_api.setting_overrides.webinspect_upload_settings:
self.webinspect_api.upload_settings()
if self.webinspect_api.setting_overrides.webinspect_upload_webmacros:
self.webinspect_api.upload_webmacros()
# if there was a provided scan policy, we've already uploaded so don't bother doing it again.
if self.webinspect_api.setting_overrides.webinspect_upload_policy and not self.webinspect_api.setting_overrides.scan_policy:
self.webinspect_api.upload_policy()
#def _scan(self, delay=10):
@CircuitBreaker(fail_max=5, reset_timeout=60)
def _scan(self):
"""
If it returns complete we are
happy and download the results files. If we enter NotRunning then something has gone wrong and we want to
exit with a failure.
:param scan_id: the id on the webinspect server for the running scan
:param delay: time between calls to Webinspect server
:return: no return but upon completion sends a "complete" message back to the queue that is waiting for it.
"""
# self.webinspect_server = self.webinspect_api.setting_overrides.endpoint
self.webinspect_api.host = self.webinspect_api.setting_overrides.endpoint
scan_complete = False
while not scan_complete:
current_status = self.webinspect_api.get_scan_status(self.scan_id)
try:
# Happy path - we completed our scan
if current_status.lower() == 'complete':
# Now let's download or export the scan artifact in two formats
self.webinspect_api.export_scan_results(
self.scan_id, 'fpr')
self.webinspect_api.export_scan_results(
self.scan_id, 'xml')
return
# TODO add json export
# The scan can sometimes go from running to not running and that is not what we want.
elif current_status.lower() == 'notrunning':
webinspectloghelper.log_error_not_running_scan()
self._stop_scan(self.scan_id)
sys.exit(ExitStatus.failure)
# This is interesting behavior and we want to log it.
# It should never be in a state besides Running, NotRunning and Complete.
elif current_status.lower() != "running":
webinspectloghelper.log_error_scan_in_weird_state(
scan_name=self.scan_id, state=current_status)
sys.exit(ExitStatus.failure)
#time.sleep(delay)
# Sometimes we are not able to get current_status and it is a None response.
except AttributeError as e:
webinspectloghelper.log_error_unrecoverable_scan(
current_status, e)
sys.exit(ExitStatus.failure)
def _stop_scan(self, scan_id):
self.webinspect_api.stop_scan(scan_id)
# below functions are for handling someone forcefully ending webbreaker.
def _exit_scan_gracefully(self, *args):
"""
called when someone ctl+c's - sends an api call to end the running scan.
:param args:
:return:
"""
Logger.app.info("Aborting!")
self.webinspect_api.stop_scan(self.scan_id)
sys.exit(ExitStatus.failure)
@contextmanager
def _termination_event_handler(self):
"""
meant to handle termination events (ctr+c and more) so that we call scan_end(scan_id) if a user decides to end the
scan.
:return:
"""
# Intercept the "please terminate" signals
original_sigint_handler = getsignal(SIGINT)
original_sigabrt_handler = getsignal(SIGABRT)
original_sigterm_handler = getsignal(SIGTERM)
for sig in (SIGABRT, SIGINT, SIGTERM):
signal(sig, self._exit_scan_gracefully)
yield # needed for context manager
# Go back to normal signal handling
signal(SIGABRT, original_sigabrt_handler)
signal(SIGINT, original_sigint_handler)
signal(SIGTERM, original_sigterm_handler)
def _webinspect_git_clone(self, cli_settings):
"""
If local file exist, it will use that file. If not, it will go to github and clone the config files
:return:
"""
try:
config_helper = Config()
etc_dir = config_helper.etc
git_dir = os.path.join(config_helper.git, '.git')
try:
if cli_settings == 'Default':
webinspectloghelper.log_info_default_settings()
elif os.path.exists(git_dir):
webinspectloghelper.log_info_updating_webinspect_configurations(
etc_dir)
check_output(['git', 'init', etc_dir])
check_output([
'git', '--git-dir=' + git_dir,
'--work-tree=' + str(config_helper.git), 'reset',
'--hard'
])
check_output([
'git', '--git-dir=' + git_dir,
'--work-tree=' + str(config_helper.git), 'pull',
'--rebase'
])
sys.stdout.flush()
elif not os.path.exists(git_dir):
webinspectloghelper.log_info_webinspect_git_clonning(
config_helper.git)
check_output([
'git', 'clone', self.config.webinspect_git,
config_helper.git
])
else:
Logger.app.error(
"No GIT Repo was declared in your config.ini, therefore nothing will be cloned!"
)
except (CalledProcessError, AttributeError) as e:
webinspectloghelper.log_webinspect_config_issue(e)
raise
except GitCommandError as e:
webinspectloghelper.log_git_access_error(
self.config.webinspect_git, e)
sys.exit(ExitStatus.failure)
except IndexError as e:
webinspectloghelper.log_config_file_unavailable(e)
sys.exit(ExitStatus.failure)
Logger.app.debug("Completed webinspect config fetch")
except TypeError as e:
webinspectloghelper.log_error_git_cloning_error(e)
