def api_post_mass_analyze(): # Abort if the API keys don't match if request.form.get('apikey', 'notatrueapikey') != API_KEY: abort(403) # Get the hostnames try: hostnames = request.form['hosts'] except KeyError: return {'error': 'scan-missing-parameters'} # Fail if it's not a valid hostname (not in DNS, not a real hostname, etc.) for host in hostnames.split(','): hostname = valid_hostname(host) or valid_hostname('www.' + host) # prepend www. if necessary if not hostname: continue # Get the site's id number try: site_id = database.select_site_id(hostname) except IOError: return {'error': 'Unable to connect to database'} # And start up a scan row = database.insert_scan(site_id) try: scan.delay(hostname, site_id, row['id']) except redis.exceptions.ConnectionError: database.update_scan_state(row['id'], STATE_ABORTED, error='redis down') return {'error': 'Unable to connect to task system'} return jsonify({'state': 'OK'})
def __start_scan(hostname, site_id): row = database.insert_scan(site_id) try: scan.delay(hostname, site_id, row['id']) except redis.exceptions.ConnectionError: database.update_scan_state(row['id'], STATE_ABORTED, error='redis down') return {'error': 'Unable to connect to task system'}
def scan(hostname: str, site_id: int, scan_id: int): try: # Once celery kicks off the task, let's update the scan state from PENDING to STARTED update_scan_state(scan_id, STATE_STARTED) # Attempt to retrieve all the resources reqs = retrieve_all(hostname) # If we can't connect at all, let's abort the test if reqs['responses']['auto'] is None: update_scan_state(scan_id, STATE_FAILED, error='site down') return # Execute each test, replacing the underscores in the function name with dashes in the test name for test in httpobs.scanner.analyzer.tests: # TODO: Get overridden expectations insert_test_result(site_id, scan_id, test.__name__.replace('_', '-'), test(reqs)) # catch the celery timeout, which will almost certainly occur in retrieve_all() except (SoftTimeLimitExceeded, TimeLimitExceeded): update_scan_state(scan_id, STATE_ABORTED, error='site unresponsive') except: # TODO: have more specific error messages e = sys.exc_info()[1] # get the error message # Print the exception to stdout if we're in dev if 'HTTPOBS_DEV' in environ: import traceback print('Error detected in: ' + hostname) traceback.print_exc() # If we are unsuccessful, close out the scan in the database update_scan_state(scan_id, STATE_FAILED, error=repr(e))
def scan(hostname: str, site_id: int, scan_id: int): try: # Once celery kicks off the task, let's update the scan state from PENDING to STARTED update_scan_state(scan_id, STATE_STARTED) # Attempt to retrieve all the resources reqs = retrieve_all(hostname) # If we can't connect at all, let's abort the test if reqs['responses']['auto'] is None: update_scan_state(scan_id, STATE_FAILED, error='site down') return # Execute each test, replacing the underscores in the function name with dashes in the test name for test in httpobs.scanner.analyzer.tests: # TODO: Get overridden expectations insert_test_result(site_id, scan_id, test.__name__.replace('_', '-'), test(reqs)) # catch the celery timeout, which will almost certainly occur in retrieve_all() except (SoftTimeLimitExceeded, TimeLimitExceeded): update_scan_state(scan_id, STATE_ABORTED, error='site unresponsive') except: # TODO: have more specific error messages e = sys.exc_info()[1] # get the error message # Print the exception to stdout if we're in dev if 'HTTPOBS_DEV' in environ: import traceback print('Error detected in: ' + hostname) traceback.print_exc() # If we are unsuccessful, close out the scan in the database update_scan_state(scan_id, STATE_FAILED, error=repr(e))
def scan(hostname: str, site_id: int, scan_id: int): try: # Once celery kicks off the task, let's update the scan state from PENDING to RUNNING update_scan_state(scan_id, STATE_RUNNING) # Get the site's cookies and headers headers = select_site_headers(hostname) # Attempt to retrieve all the resources reqs = retrieve_all(hostname, cookies=headers['cookies'], headers=headers['headers']) # If we can't connect at all, let's abort the test if reqs['responses']['auto'] is None: update_scan_state(scan_id, STATE_FAILED, error='site down') return # Execute each test, replacing the underscores in the function name with dashes in the test name # TODO: Get overridden expectations insert_test_results(site_id, scan_id, [test(reqs) for test in tests], sanitize_headers(reqs['responses']['auto'].headers), reqs['responses']['auto'].status_code) # catch the celery timeout, which will almost certainly occur in retrieve_all() except SoftTimeLimitExceeded: update_scan_state(scan_id, STATE_ABORTED, error='site unresponsive') except (TimeLimitExceeded, WorkerLostError, WorkerShutdown, WorkerTerminate): raise # the database is down, oh no! except IOError: print('database down, aborting scan on {hostname}'.format(hostname=hostname), file=sys.stderr) except: # TODO: have more specific error messages e = sys.exc_info()[1] # get the error message # If we are unsuccessful, close out the scan in the database update_scan_state(scan_id, STATE_FAILED, error=repr(e)) # Print the exception to stderr if we're in dev if DEVELOPMENT_MODE: import traceback print('Error detected in scan for : ' + hostname) traceback.print_exc(file=sys.stderr)