def api_post_mass_analyze():
# Abort if the API keys don't match
if request.form.get('apikey', 'notatrueapikey') != API_KEY:
abort(403)
# Get the hostnames
try:
hostnames = request.form['hosts']
except KeyError:
return {'error': 'scan-missing-parameters'}
# Fail if it's not a valid hostname (not in DNS, not a real hostname, etc.)
for host in hostnames.split(','):
hostname = valid_hostname(host) or valid_hostname('www.' + host) # prepend www. if necessary
if not hostname:
continue
# Get the site's id number
try:
site_id = database.select_site_id(hostname)
except IOError:
return {'error': 'Unable to connect to database'}
# And start up a scan
row = database.insert_scan(site_id)
try:
scan.delay(hostname, site_id, row['id'])
except redis.exceptions.ConnectionError:
database.update_scan_state(row['id'], STATE_ABORTED, error='redis down')
return {'error': 'Unable to connect to task system'}
return jsonify({'state': 'OK'})
def __start_scan(hostname, site_id):
row = database.insert_scan(site_id)
try:
scan.delay(hostname, site_id, row['id'])
except redis.exceptions.ConnectionError:
database.update_scan_state(row['id'], STATE_ABORTED, error='redis down')
return {'error': 'Unable to connect to task system'}
def scan(hostname: str, site_id: int, scan_id: int):
try:
# Once celery kicks off the task, let's update the scan state from PENDING to STARTED
update_scan_state(scan_id, STATE_STARTED)
# Attempt to retrieve all the resources
reqs = retrieve_all(hostname)
# If we can't connect at all, let's abort the test
if reqs['responses']['auto'] is None:
update_scan_state(scan_id, STATE_FAILED, error='site down')
return
# Execute each test, replacing the underscores in the function name with dashes in the test name
for test in httpobs.scanner.analyzer.tests:
# TODO: Get overridden expectations
insert_test_result(site_id, scan_id, test.__name__.replace('_', '-'), test(reqs))
# catch the celery timeout, which will almost certainly occur in retrieve_all()
except (SoftTimeLimitExceeded, TimeLimitExceeded):
update_scan_state(scan_id, STATE_ABORTED, error='site unresponsive')
except:
# TODO: have more specific error messages
e = sys.exc_info()[1] # get the error message
# Print the exception to stdout if we're in dev
if 'HTTPOBS_DEV' in environ:
import traceback
print('Error detected in: ' + hostname)
traceback.print_exc()
# If we are unsuccessful, close out the scan in the database
update_scan_state(scan_id, STATE_FAILED, error=repr(e))
def scan(hostname: str, site_id: int, scan_id: int):
try:
# Once celery kicks off the task, let's update the scan state from PENDING to STARTED
update_scan_state(scan_id, STATE_STARTED)
# Attempt to retrieve all the resources
reqs = retrieve_all(hostname)
# If we can't connect at all, let's abort the test
if reqs['responses']['auto'] is None:
update_scan_state(scan_id, STATE_FAILED, error='site down')
return
# Execute each test, replacing the underscores in the function name with dashes in the test name
for test in httpobs.scanner.analyzer.tests:
# TODO: Get overridden expectations
insert_test_result(site_id, scan_id,
test.__name__.replace('_', '-'), test(reqs))
# catch the celery timeout, which will almost certainly occur in retrieve_all()
except (SoftTimeLimitExceeded, TimeLimitExceeded):
update_scan_state(scan_id, STATE_ABORTED, error='site unresponsive')
except:
# TODO: have more specific error messages
e = sys.exc_info()[1] # get the error message
# Print the exception to stdout if we're in dev
if 'HTTPOBS_DEV' in environ:
import traceback
print('Error detected in: ' + hostname)
traceback.print_exc()
# If we are unsuccessful, close out the scan in the database
update_scan_state(scan_id, STATE_FAILED, error=repr(e))
def scan(hostname: str, site_id: int, scan_id: int):
try:
# Once celery kicks off the task, let's update the scan state from PENDING to RUNNING
update_scan_state(scan_id, STATE_RUNNING)
# Get the site's cookies and headers
headers = select_site_headers(hostname)
# Attempt to retrieve all the resources
reqs = retrieve_all(hostname, cookies=headers['cookies'], headers=headers['headers'])
# If we can't connect at all, let's abort the test
if reqs['responses']['auto'] is None:
update_scan_state(scan_id, STATE_FAILED, error='site down')
return
# Execute each test, replacing the underscores in the function name with dashes in the test name
# TODO: Get overridden expectations
insert_test_results(site_id,
scan_id,
[test(reqs) for test in tests],
sanitize_headers(reqs['responses']['auto'].headers),
reqs['responses']['auto'].status_code)
# catch the celery timeout, which will almost certainly occur in retrieve_all()
except SoftTimeLimitExceeded:
update_scan_state(scan_id, STATE_ABORTED, error='site unresponsive')
except (TimeLimitExceeded, WorkerLostError, WorkerShutdown, WorkerTerminate):
raise
# the database is down, oh no!
except IOError:
print('database down, aborting scan on {hostname}'.format(hostname=hostname), file=sys.stderr)
except:
# TODO: have more specific error messages
e = sys.exc_info()[1] # get the error message
# If we are unsuccessful, close out the scan in the database
update_scan_state(scan_id, STATE_FAILED, error=repr(e))
# Print the exception to stderr if we're in dev
if DEVELOPMENT_MODE:
import traceback
print('Error detected in scan for : ' + hostname)
traceback.print_exc(file=sys.stderr)