def analyze_response(self, ftarget, response):
""" Analyze the results of the request and return the info gathered.
ftargeet FuzzyTarget object.
response requests.Response object.
returns Result object.
raises TypeError or AttributeError when non requests.Response is given as response.
"""
#FIXME: Clarify with alex: !function is a mess, response is of type
# text or non-text, trying to read blah blah
result_dic = {}
check_type_list = ftarget.payload.check_type_list
if self.hadoop_reporting:
logger.info("Response is of type %s for target %s.",
response.__class__.__name__, ftarget)
worthy = parse_worthy(response,
hadoop_reporting=self.hadoop_reporting)
if worthy:
logger.info("FuzzyTarget %s looks worth checking for vulnerabilities.",
ftarget)
else:
logger.info("Response deemed non-parse-worthy. Setting all checks "
"in result_dic to False for %s", ftarget)
return self._make_failed_result(ftarget)
result_dic = self._run_checks(response, result_dic, check_type_list)
return Result(ftarget, result_dic)
def parse(self, stay_in_scope=True, max_links=10):
for target, response in self.results:
# skip 40X replies and strings (i.e. failed requests)
logger.info("Attempting to parse %s", target)
try:
response.raise_for_status()
except (HTTPError, AttributeError) as exc: # only exception type we care about from requests.Response
logger.debug("Failed request.", exc_info=True)
continue
if parse_worthy(response, content_type_match="text/html",
hadoop_reporting=True):
logger.info("pase_worthy function tells us to parse")
else:
logger.info("pase_worthy function tells us not to try"
" parsing")
continue
logger.info("Finding post requests on page %s", response.url)
#FIXME: !this doesn't stay in scope?
post_request_targets = find_post_requests(target=response.url,
response_text=response.text)
for target_post in post_request_targets:
ct_post = CrawlTarget(target_post.url)
ct_post.__dict__ = target_post.__dict__
ct_post.status = "unfetched"
self.add_target(ct_post)
links = self.parse_response(response, stay_in_scope=stay_in_scope,
max_links=max_links)
for link in links:
ct_link = CrawlTarget(unicode(link))
self.add_target(ct_link)
if stay_in_scope:
self.filter_targets_by_scope()
logger.info("Finished attempted parsing for %s", target)
def parse(self, stay_in_scope=True, max_links=10):
for target, response in self.results:
# skip 40X replies and strings (i.e. failed requests)
logger.info("Attempting to parse %s", target)
try:
response.raise_for_status()
except (
HTTPError, AttributeError
) as exc: # only exception type we care about from requests.Response
logger.debug("Failed request.", exc_info=True)
continue
if parse_worthy(response,
content_type_match="text/html",
hadoop_reporting=True):
logger.info("pase_worthy function tells us to parse")
else:
logger.info("pase_worthy function tells us not to try"
" parsing")
continue
logger.info("Finding post requests on page %s", response.url)
#FIXME: !this doesn't stay in scope?
post_request_targets = find_post_requests(
target=response.url, response_text=response.text)
for target_post in post_request_targets:
ct_post = CrawlTarget(target_post.url)
ct_post.__dict__ = target_post.__dict__
ct_post.status = "unfetched"
self.add_target(ct_post)
links = self.parse_response(response,
stay_in_scope=stay_in_scope,
max_links=max_links)
for link in links:
ct_link = CrawlTarget(unicode(link))
self.add_target(ct_link)
if stay_in_scope:
self.filter_targets_by_scope()
logger.info("Finished attempted parsing for %s", target)
