def process_item(self, item, spider):
response = item['resp']
meta = response.meta
item = vuln()
payload = meta['payload']
delim = meta['delim']
resp_url = response.url
body = response.body
mismatch = False
orig_payload = payload.replace(delim, '').replace(';9', '') # xss char payload
# Regex: ( ) mean group 1 is within the parens, . means any char,
# {1,50} means match any char 0 to 72 times, 72 chosen because double URL encoding
# ? makes the search nongreedy so it stops after hitting its limits
full_match = '%s.*?%s' % (delim, delim)
# matches with semicolon which sometimes cuts results off
sc_full_match = '%s.*?%s;9' % (delim, delim)
chars_between_delims = '%s(.*?)%s' % (delim, delim)
re_matches = sorted([(m.start(), m.group()) for m in re.finditer(full_match, body)])
if re_matches:
scolon_matches = sorted([(m.start(), m.group()) for m in re.finditer(sc_full_match, body)])
lxml_match_data = self.get_lxml_matches(full_match, body, resp_url, delim)
if lxml_match_data:
if len(re_matches) != len(lxml_match_data):
spider.log('Mismatch in lxml parsed injections and regex parsed injections. Higher chance of false positive for %s' % resp_url)
mismatch = True
#zipped = zip(re_matches, lxml_match_data)
#for z in zipped:
# print z
inj_data = self.combine_inj_data(lxml_match_data, re_matches, scolon_matches, delim)
for i in inj_data:
########## XSS LOGIC #############
item = self.xss_logic(i, meta, resp_url, body)
if item:
if item['xss_place'] == 'url':
item = self.url_item_filtering(item, spider)
if mismatch:
item['error'] = 'Mismatch: html parsed vs regex parsed injections, %d vs %d. Higher chance of false positive' % (len(injections), len(full_matches))
self.write_to_file(item, spider)
return item
# Catch all test payload chars no delims
# Catches DB errors
#pl_lines_found = self.payloaded_lines(body, orig_payload)
#if pl_lines_found:
# print '\n\n\n LAST RESORT\n %s %s %s \n\n\n' % (resp_url, meta['xss_place'], meta['xss_param'])
# return self.make_item(meta, resp_url, pl_lines_found, orig_payload)
raise DropItem('No XSS vulns in %s. type = %s, %s, %s'% (resp_url, meta['xss_place'], meta['xss_param'], meta['payload']))
def make_item(self, meta, resp_url, lines, unfiltered):
item = vuln()
''' Create the vuln item '''
item['lines'] = lines.strip()
item['xss_payload'] = meta['payload'] + ';'
item['unfiltered'] = unfiltered
item['xss_param'] = meta['xss_param']
item['xss_place'] = meta['xss_place']
item['orig_url'] = meta['orig_url']
item['resp_url'] = resp_url
if 'POST_to' in meta:
item['POST_to'] = meta['POST_to']
return item
def make_item(self, meta, resp_url, line, unfiltered):
item = vuln()
''' Create the vuln item '''
if isinstance(line, str):
item['line'] = line
else:
item['line'] = '\n'.join(line)
item['xss_payload'] = meta['payload']
item['unfiltered'] = unfiltered
item['xss_param'] = meta['xss_param']
item['xss_place'] = meta['xss_place']
item['orig_url'] = meta['orig_url']
item['resp_url'] = resp_url
if 'POST_to' in meta:
item['POST_to'] = meta['POST_to']
return item
def make_item(self, meta, resp_url, line, unfiltered):
item = vuln()
''' Create the vuln item '''
if isinstance(line, str):
item['line'] = line
else:
item['line'] = '\n'.join(line)
item['xss_payload'] = meta['payload']
item['unfiltered'] = unfiltered
item['xss_param'] = meta['xss_param']
item['xss_place'] = meta['xss_place']
item['orig_url'] = meta['orig_url']
item['resp_url'] = resp_url
if 'POST_to' in meta:
item['POST_to'] = meta['POST_to']
return item
def make_item(self, meta, resp_url, line, unfiltered, sugg_payloads):
''' Create the vuln item '''
item = vuln()
if isinstance(line, str):
item['line'] = line
else:
item['line'] = '\n'.join(line)
item['xss_payload'] = meta['payload']
item['unfiltered'] = unfiltered
item['xss_param'] = meta['xss_param']
item['xss_place'] = meta['xss_place']
item['orig_url'] = meta['orig_url']
item['resp_url'] = resp_url
if sugg_payloads:
item['sugg_payloads'] = ', '.join(sugg_payloads)
if 'POST_to' in meta:
item['POST_to'] = meta['POST_to']
# Just make sure one of the options has been set
if item['unfiltered']:
return item
def make_item(self, meta, resp_url, line, unfiltered, sugg_payloads):
''' Create the vuln item '''
item = vuln()
if isinstance(line, str):
item['line'] = line
else:
item['line'] = '\n'.join(line)
item['xss_payload'] = meta['payload']
item['unfiltered'] = unfiltered
item['xss_param'] = meta['xss_param']
item['xss_place'] = meta['xss_place']
item['orig_url'] = meta['orig_url']
item['resp_url'] = resp_url
if sugg_payloads:
item['sugg_payloads'] = ', '.join(sugg_payloads)
if 'POST_to' in meta:
item['POST_to'] = meta['POST_to']
# Just make sure one of the options has been set
if item['unfiltered']:
return item
def make_item(self, meta, resp_url, line, unfiltered, sugg_payloads):
""" Create the vuln item """
item = vuln()
if isinstance(line, str):
item["line"] = line
else:
item["line"] = "\n".join(line)
item["xss_payload"] = meta["payload"]
item["unfiltered"] = unfiltered
item["xss_param"] = meta["xss_param"]
item["xss_place"] = meta["xss_place"]
item["orig_url"] = meta["orig_url"]
item["resp_url"] = resp_url
if sugg_payloads:
item["sugg_payloads"] = ", ".join(sugg_payloads)
if "POST_to" in meta:
item["POST_to"] = meta["POST_to"]
# Just make sure one of the options has been set
if item["unfiltered"]:
return item
def process_item(self, item, spider):
response = item['resp']
meta = response.meta
item = vuln()
payload = meta['payload']
delim = meta['delim']
resp_url = response.url
body = response.body.lower()
mismatch = False
error = None
orig_payload = payload.replace(delim,
'').replace(';9',
'') # xss char payload
# Regex: ( ) mean group 1 is within the parens, . means any char,
# {1,50} means match any char 0 to 72 times, 72 chosen because double URL encoding
# ? makes the search nongreedy so it stops after hitting its limits
#full_match = '%s.*?%s' % (delim, delim)
full_match = '%s.{0,80}?%s' % (delim, delim)
# matches with semicolon which sometimes cuts results off
sc_full_match = '%s.{0,80}?%s;9' % (delim, delim)
#chars_between_delims = '%s(.*?)%s' % (delim, delim)
chars_between_delims = '%s(.{0,80}?)%s' % (delim, delim)
re_matches = sorted([(m.start(), m.group())
for m in re.finditer(full_match, body)])
if re_matches:
scolon_matches = sorted([(m.start(), m.group())
for m in re.finditer(sc_full_match, body)
])
lxml_injs = self.get_lxml_matches(full_match, body, resp_url,
delim)
if lxml_injs:
err = None
if len(re_matches) != len(lxml_injs):
spider.log(
'Error: mismatch in injections found by lxml and regex. Higher chance of false positive for %s'
% resp_url)
error = 'Error: mismatch in injections found by lxml and regex; higher chance of false positive'
mismatch = True
inj_data = self.combine_regex_lxml(lxml_injs, re_matches,
scolon_matches, body,
mismatch)
# If mismatch is True, then "for offset in sorted(inj_data)" will fail with TypeError
try:
for offset in sorted(inj_data):
########## XSS LOGIC #############
item = self.xss_logic(inj_data[offset], meta, resp_url,
error)
if item:
if item['xss_place'] == 'url':
item = self.url_item_filtering(item, spider)
if mismatch:
item[
'error'] = 'Mismatch: html parsed vs regex parsed injections, %d vs %d. Higher chance of false positive' % (
len(injections), len(full_matches))
self.write_to_file(item, spider)
return item
except TypeError:
if mismatch:
return
else:
raise
# No lxml_match_data
else:
spider.log('Error: regex found injection, but lxml did not')
# Catch all test payload chars no delims
# Catches DB errors
pl_lines_found = self.payloaded_lines(body, orig_payload)
if pl_lines_found:
item = self.make_item(meta, resp_url, pl_lines_found, orig_payload,
None)
if item:
if item['xss_place'] == 'url':
item = self.url_item_filtering(item, spider)
item[
'error'] = 'Payload delims do not surround this injection point. Found via search for entire payload.'
self.write_to_file(item, spider)
raise DropItem('No XSS vulns in %s. type = %s, %s' %
(resp_url, meta['xss_place'], meta['xss_param']))
def process_item(self, item, spider):
response = item['resp']
meta = response.meta
item = vuln()
payload = meta['payload']
delim = meta['delim']
resp_url = response.url
body = response.body.lower()
mismatch = False
error = None
orig_payload = payload.replace(delim, '').replace(';9', '') # xss char payload
# Regex: ( ) mean group 1 is within the parens, . means any char,
# {1,50} means match any char 0 to 72 times, 72 chosen because double URL encoding
# ? makes the search nongreedy so it stops after hitting its limits
#full_match = '%s.*?%s' % (delim, delim)
full_match = '%s.{0,80}?%s' % (delim, delim)
# matches with semicolon which sometimes cuts results off
sc_full_match = '%s.{0,80}?%s;9' % (delim, delim)
#chars_between_delims = '%s(.*?)%s' % (delim, delim)
chars_between_delims = '%s(.{0,80}?)%s' % (delim, delim)
re_matches = sorted([(m.start(), m.group()) for m in re.finditer(full_match, body)])
if re_matches:
scolon_matches = sorted([(m.start(), m.group()) for m in re.finditer(sc_full_match, body)])
lxml_injs = self.get_lxml_matches(full_match, body, resp_url, delim)
if lxml_injs:
err = None
if len(re_matches) != len(lxml_injs):
spider.log('Error: mismatch in injections found by lxml and regex. Higher chance of false positive for %s' % resp_url)
error = 'Error: mismatch in injections found by lxml and regex; higher chance of false positive'
mismatch = True
inj_data = self.combine_regex_lxml(lxml_injs, re_matches, scolon_matches, body, mismatch)
# If mismatch is True, then "for offset in sorted(inj_data)" will fail with TypeError
try:
for offset in sorted(inj_data):
########## XSS LOGIC #############
item = self.xss_logic(inj_data[offset], meta, resp_url, error)
if item:
if item['xss_place'] == 'url':
item = self.url_item_filtering(item, spider)
if mismatch:
item['error'] = 'Mismatch: html parsed vs regex parsed injections, %d vs %d. Higher chance of false positive' % (len(injections), len(full_matches))
self.write_to_file(item, spider)
return item
except TypeError:
if mismatch:
return
else:
raise
# No lxml_match_data
else:
spider.log('Error: regex found injection, but lxml did not')
# Catch all test payload chars no delims
# Catches DB errors
pl_lines_found = self.payloaded_lines(body, orig_payload)
if pl_lines_found:
item = self.make_item(meta, resp_url, pl_lines_found, orig_payload, None)
if item:
if item['xss_place'] == 'url':
item = self.url_item_filtering(item, spider)
item['error'] = 'Payload delims do not surround this injection point. Found via search for entire payload.'
self.write_to_file(item, spider)
raise DropItem('No XSS vulns in %s. type = %s, %s' % (resp_url, meta['xss_place'], meta['xss_param']))
def process_item(self, item, spider):
response = item['resp']
item = vuln()
xss_type = response.meta['type']
orig_url = response.meta['orig_url']
injections = response.meta['injections']
quote_enclosure = response.meta['quote']
inj_point = response.meta['inj_point']
resp_url = response.url
body = response.body
# Regex: ( ) mean group 1 is within the parens, . means any char, {1,75} means match any char 1 to 25 times
#chars_between_delims = '%s(.{1,75}?)%s' % (self.test_str, self.test_str)
chars_between_delims = '%s(.{0,50}?)%s' % (self.test_str,
self.test_str)
inj_num = len(injections)
mismatch = False
if xss_type == 'form':
POST_to = response.meta['POST_to']
else:
POST_to = None
orig_payload = response.meta['payload'].strip(
self.test_str) # xss char payload
escaped_payload = self.unescape_payload(orig_payload)
break_tag_chars = set(['>', '<', '(', ')'])
break_attr_chars = set([quote_enclosure, '(', ')'])
break_js_chars = set(['"', "'", '(', ')'])
matches = re.findall(chars_between_delims, body)
if matches:
xss_num = len(matches)
if xss_num != inj_num:
err = (
'Mismatch between harmless injection count and payloaded injection count: %d vs %d, increased chance of false positive'
% (inj_num, xss_num))
item['error'] = err
for idx, match in enumerate(matches):
unfiltered_chars = self.get_unfiltered_chars(
match, escaped_payload)
if unfiltered_chars:
try:
line, tag, attr, attr_val = spider.parse_injections(
injections[idx])
except IndexError:
mismatch = True
# Mismatch in num of test injections and num of payloads found
line, tag, attr, attr_val = 'Unknown', 'Unknown', None, None
joined_chars = ''.join(unfiltered_chars)
chars = set(joined_chars)
line_html = self.get_inj_line(body, match)
###### XSS RULES ########
# If there's more XSS matches than harmless injections, we still want to check for the most dangerous characters
# May see some false positives here, but better than false negatives
if mismatch == True:
if '>' in escaped_payload and '<' in escaped_payload:
if '<' in joined_chars and '>' in joined_chars:
item = self.make_item(joined_chars, xss_type,
orig_payload, tag,
orig_url, inj_point,
line_html, POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# Redirect
if 'javascript:prompt(99)' == joined_chars.lower(
): # redir
item = self.make_item(joined_chars, xss_type,
orig_payload, tag, orig_url,
inj_point, line_html, POST_to,
item)
item = self.url_item_filtering(item, spider)
return item
# JS breakout
if self.js_pld == escaped_payload: #js chars
if break_js_chars.issubset(chars):
item = self.make_item(joined_chars, xss_type,
orig_payload, tag, orig_url,
inj_point, line_html,
POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# Attribute breakout
if attr:
if quote_enclosure in escaped_payload:
if break_attr_chars.issubset(chars):
item = self.make_item(joined_chars, xss_type,
orig_payload, tag,
orig_url, inj_point,
line_html, POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# Tag breakout
else:
if '<' and '>' in escaped_payload:
if break_tag_chars.issubset(chars):
item = self.make_item(joined_chars, xss_type,
orig_payload, tag,
orig_url, inj_point,
line_html, POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# Check the entire body for exact match
# Escape out all the special regex characters to search for the payload in the html body
re_payload = escaped_payload.replace('(', '\(').replace(
')', '\)').replace('"', '\\"').replace("'", "\\'")
re_payload = re_payload.replace('{', '\{').replace('}', '\}').replace(
']', '\]').replace('[', '\[')
re_payload = '.{1}?' + re_payload
full_matches = re.findall(re_payload, body)
for f in full_matches:
unescaped_match = ''.join(
self.get_unfiltered_chars(f, escaped_payload))
if unescaped_match == escaped_payload:
#if '\\' == unescaped_match[0]:
# continue
item[
'error'] = 'Response passed injection point specific search without success, checked for exact payload match in body (higher chance of false positive here)'
item['line'] = self.get_inj_line(body, f)
item['xss_payload'] = orig_payload
item['unfiltered'] = escaped_payload
item['inj_point'] = inj_point
item['xss_type'] = xss_type
item['url'] = orig_url
if POST_to:
item['POST_to'] = POST_to
return item
#for k in item:
# print k, item[k]
# In case it slips by all of the filters, then we move on
raise DropItem('No XSS vulns in %s' % resp_url)
def process_item(self, item, spider):
response = item['resp']
item = vuln()
xss_type = response.meta['type']
orig_url = response.meta['orig_url']
injections = response.meta['injections']
quote_enclosure = response.meta['quote']
inj_point = response.meta['inj_point']
resp_url = response.url
body = response.body
# Regex: ( ) mean group 1 is within the parens, . means any char, {1,75} means match any char 1 to 25 times
#chars_between_delims = '%s(.{1,75}?)%s' % (self.test_str, self.test_str)
chars_between_delims = '%s(.{0,50}?)%s' % (self.test_str, self.test_str)
inj_num = len(injections)
mismatch = False
if xss_type == 'form':
POST_to = response.meta['POST_to']
else:
POST_to = None
orig_payload = response.meta['payload'].strip(self.test_str) # xss char payload
escaped_payload = self.unescape_payload(orig_payload)
break_tag_chars = set(['>', '<', '(', ')'])
break_attr_chars = set([quote_enclosure, '(', ')'])
break_js_chars = set(['"', "'", '(', ')'])
matches = re.findall(chars_between_delims, body)
if matches:
xss_num = len(matches)
if xss_num != inj_num:
err = ('Mismatch between harmless injection count and payloaded injection count: %d vs %d, increased chance of false positive' % (inj_num, xss_num))
item['error'] = err
for idx, match in enumerate(matches):
unfiltered_chars = self.get_unfiltered_chars(match, escaped_payload)
if unfiltered_chars:
try:
line, tag, attr, attr_val = spider.parse_injections(injections[idx])
except IndexError:
mismatch = True
# Mismatch in num of test injections and num of payloads found
line, tag, attr, attr_val = 'Unknown', 'Unknown', None, None
joined_chars = ''.join(unfiltered_chars)
chars = set(joined_chars)
line_html = self.get_inj_line(body, match)
###### XSS RULES ########
# If there's more XSS matches than harmless injections, we still want to check for the most dangerous characters
# May see some false positives here, but better than false negatives
if mismatch == True:
if '>' in escaped_payload and '<' in escaped_payload:
if '<' in joined_chars and '>' in joined_chars:
item = self.make_item(joined_chars, xss_type, orig_payload, tag, orig_url, inj_point, line_html, POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# Redirect
if 'javascript:prompt(99)' == joined_chars.lower(): # redir
item = self.make_item(joined_chars, xss_type, orig_payload, tag, orig_url, inj_point, line_html, POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# JS breakout
if self.js_pld == escaped_payload: #js chars
if break_js_chars.issubset(chars):
item = self.make_item(joined_chars, xss_type, orig_payload, tag, orig_url, inj_point, line_html, POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# Attribute breakout
if attr:
if quote_enclosure in escaped_payload:
if break_attr_chars.issubset(chars):
item = self.make_item(joined_chars, xss_type, orig_payload, tag, orig_url, inj_point, line_html, POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# Tag breakout
else:
if '<' and '>' in escaped_payload:
if break_tag_chars.issubset(chars):
item = self.make_item(joined_chars, xss_type, orig_payload, tag, orig_url, inj_point, line_html, POST_to, item)
item = self.url_item_filtering(item, spider)
return item
# Check the entire body for exact match
# Escape out all the special regex characters to search for the payload in the html body
re_payload = escaped_payload.replace('(', '\(').replace(')', '\)').replace('"', '\\"').replace("'", "\\'")
re_payload = re_payload.replace('{', '\{').replace('}', '\}').replace(']', '\]').replace('[', '\[')
re_payload = '.{1}?'+re_payload
full_matches = re.findall(re_payload, body)
for f in full_matches:
unescaped_match = ''.join(self.get_unfiltered_chars(f, escaped_payload))
if unescaped_match == escaped_payload:
#if '\\' == unescaped_match[0]:
# continue
item['error'] = 'Response passed injection point specific search without success, checked for exact payload match in body (higher chance of false positive here)'
item['line'] = self.get_inj_line(body, f, item)
item['xss_payload'] = orig_payload
item['unfiltered'] = escaped_payload
item['inj_point'] = inj_point
item['xss_type'] = xss_type
item['url'] = orig_url
if POST_to:
item['POST_to'] = POST_to
return item
#for k in item:
# print k, item[k]
# In case it slips by all of the filters, then we move on
raise DropItem('No XSS vulns in %s' % resp_url)