def process(self, request, response, report):
global _enchant_available
if response.is_html and _enchant_available:
doc = HtmlHelper(response.content)
doc.strip_comments()
doc.strip_elements(('script', 'style'))
words = {}
with self.sync_lock:
for txt in doc.get_text():
self._check(txt, words)
for txt in doc.get_attribute('title'):
self._check(txt[2], words)
for txt in doc.get_attribute('alt'):
self._check(txt[2], words)
for e in doc.get_elements('meta'):
names = [n for n in e.get_attribute('name')]
if len(names) > 0:
name = names[0][2].lower()
if name == 'description' or name == 'keywords':
content = [c for c in e.get_attribute('content')]
if len(content) > 0:
self._check(content[0][2], words)
if len(words) > 0:
keys = list(words.keys())
keys.sort()
for k in keys:
report.add_message('Word: [{0}] x {1} ({2})'.format(words[k][0], words[k][1], words[k][2]))
def process(self, request, response, report):
if response.is_html:
doc = HtmlHelper(response.content)
doc.strip_comments()
doc.strip_elements(('script', 'style'))
th = TextHelper()
for txt in doc.get_text():
th.append(txt)
if len(th) > 0:
twrd = float(th.word_count())
tsnt = float(th.sentence_count())
tsyl = float(th.syllable_count())
fkre = 206.835 - 1.015 * (twrd / tsnt) - 84.6 * (tsyl / twrd)
with self.sync_lock:
self.count += 1
self.total += fkre
if self.min == None:
self.min = fkre
else:
self.min = min(self.min, fkre)
if self.max == None:
self.max = fkre
else:
self.max = max(self.max, fkre)
if fkre < self.threshold:
report.add_message('Readability: [{1:.2f}]'.format(str(request), fkre))
def begin(self, report):
if self.spell_checker:
report.add_message('Language: {0}'.format(self.language))
if self.dictionary:
report.add_message('Using custom dictionary [{0}]'.format(self.dictionary))
else:
report.add_error('pyenchant not available')
def process(self, request, response, report):
if response.is_html:
doc = HtmlHelper(response.content)
doc.strip_comments()
doc.strip_elements(('script', 'style'))
th = TextHelper()
for txt in doc.get_text():
th.append(txt)
if len(th) > 0:
twrd = float(th.word_count())
tsnt = float(th.sentence_count())
tsyl = float(th.syllable_count())
fkre = 206.835 - 1.015 * (twrd / tsnt) - 84.6 * (tsyl / twrd)
with self.sync_lock:
self.count += 1
self.total += fkre
if self.min == None:
self.min = fkre
else:
self.min = min(self.min, fkre)
if self.max == None:
self.max = fkre
else:
self.max = max(self.max, fkre)
if fkre < self.threshold:
report.add_message('Readability: [{1:.2f}]'.format(
str(request), fkre))
def process(self, request, response, report):
global _enchant_available
if response.is_html and _enchant_available:
doc = HtmlHelper(response.content)
doc.strip_comments()
doc.strip_elements(('script', 'style'))
words = {}
with self.sync_lock:
for txt in doc.get_text():
self._check(txt, words)
for txt in doc.get_attribute('title'):
self._check(txt[2], words)
for txt in doc.get_attribute('alt'):
self._check(txt[2], words)
for e in doc.get_elements('meta'):
names = [n for n in e.get_attribute('name')]
if len(names) > 0:
name = names[0][2].lower()
if name == 'description' or name == 'keywords':
content = [c for c in e.get_attribute('content')]
if len(content) > 0:
self._check(content[0][2], words)
if len(words) > 0:
keys = list(words.keys())
keys.sort()
for k in keys:
report.add_message('Word: [{0}] x {1} ({2})'.format(
words[k][0], words[k][1], words[k][2]))
def process(self, request, response, report):
if request.source == self.name:
err = False
if response.status >= 500:
err = True
report.add_warning('Possible SQL injection')
elif self.xss.search(response.content):
err = True
report.add_warning('Possible XSS')
if 'vector' in request.meta and err:
if request.meta['vector'] == 'post_data':
report.add_message('Post data: {0}'.format(
request.post_data))
elif request.meta['vector'] == 'headers':
report.add_message('Request headers: {0}'.format(
request.headers))
elif response.is_html and response.status < 500:
# Don't attack error pages - can't tell if it worked without matching against known database error text
doc = HtmlHelper(response.content)
for atk in self.attacks:
if self.quick:
self._inject_all(request, doc, atk)
else:
self._inject_each(request, doc, atk)
def begin(self, report):
if self.spell_checker:
report.add_message('Language: {0}'.format(self.language))
if self.dictionary:
report.add_message('Using custom dictionary [{0}]'.format(
self.dictionary))
else:
report.add_error('pyenchant not available')
def _report(self, report, urls):
out = list(urls)
if len(out) > 0:
out.sort()
for url in out:
if url.count(' ') > 0:
report.add_message('-> [{0}] *Unencoded'.format(url))
else:
report.add_message('-> [{0}]'.format(url))
def _report(self, report, urls):
out = list(urls)
if len(out) > 0:
out.sort()
for url in out:
if url.count(' ') > 0:
report.add_message('-> [{0}] *Unencoded'.format(url))
else:
report.add_message('-> [{0}]'.format(url))
def process(self, request, response, report):
if response.is_html:
doc = HtmlHelper(response.content)
for comment in doc.get_comments():
c = comment.strip()
if c.startswith('[if') and c.endswith('<![endif]'):
# Ignore IE conditional comments
pass
else:
report.add_message('Comment:\t{0}'.format(re.sub('\r?\n', '\n\t\t\t\t', c, re.MULTILINE)))
def process(self, request, response, report):
if response.is_html and request.protocol.lower() == 'https':
doc = HtmlHelper(response.content)
for e, a, v in doc.get_attribute('src'): #img, script
if v.lower().startswith('http:'):
report.add_message('{0}'.format(v))
for e, a, v in doc.get_attribute('href', 'link'):
if v.lower().startswith('http:'):
report.add_message('{0}'.format(v))
def process(self, request, response, report):
if response.is_html:
doc = HtmlHelper(response.content)
for comment in doc.get_comments():
c = comment.strip()
if c.startswith('[if') and c.endswith('<![endif]'):
# Ignore IE conditional comments
pass
else:
report.add_message('Comment:\t{0}'.format(
re.sub('\r?\n', '\n\t\t\t\t', c, re.MULTILINE)))
def process(self, request, response, report):
if response.is_html and request.protocol.lower() == 'https':
doc = HtmlHelper(response.content)
for e, a, v in doc.get_attribute('src'): #img, script
if v.lower().startswith('http:'):
report.add_message('{0}'.format(v))
for e, a, v in doc.get_attribute('href', 'link'):
if v.lower().startswith('http:'):
report.add_message('{0}'.format(v))
def process(self, request, response, report):
for rx in self.expressions.items():
inv_h = inv_b = False
if rx[0][0] == '^':
inv_h = True
elif rx[0][0] == '_':
inv_b = True
if inv_h:
if not rx[1].search(str(response.headers)):
report.add_message(
'Filter: [{0}] not found in headers'.format(rx[0]))
elif not inv_b:
mtchs = rx[1].finditer(str(response.headers))
for mtch in mtchs:
report.add_message(
'Filter: [{0}] found: [{1}] in headers'.format(
rx[0], mtch.group()))
if response.is_html:
if inv_b:
if not rx[1].search(str(response.content)):
report.add_message('Filter: [{0}] not found'.format(
rx[0]))
elif not inv_h:
mtchs = rx[1].finditer(response.content)
for mtch in mtchs:
report.add_message('Filter: [{0}] found: [{1}]'.format(
rx[0], mtch.group()))
def process(self, request, response, report):
global _tidy_available
#TODO: Hash errors and don't log duplicate error sets (just a reference)
if response.is_html and _tidy_available:
try:
doc, err = tidy_document(response.content, options=self.options)
except:
report.add_error('Unable to parse response')
else:
l = err.splitlines()
if len(l) > 0:
for e in l:
report.add_message('{0}'.format(re.sub('^line\\b', 'Line', e)))
report.add_message('Total: {0}'.format(len(l)))
def process(self, request, response, report):
global _tidy_available
#TODO: Hash errors and don't log duplicate error sets (just a reference)
if response.is_html and _tidy_available:
try:
doc, err = tidy_document(response.content,
options=self.options)
except:
report.add_error('Unable to parse response')
else:
l = err.splitlines()
if len(l) > 0:
for e in l:
report.add_message('{0}'.format(
re.sub('^line\\b', 'Line', e)))
report.add_message('Total: {0}'.format(len(l)))
def process(self, request, response, report):
if response.is_html and response.status < 300:
m = hashlib.sha1()
m.update(response.content.encode())
h = m.hexdigest()
dup = False
with self.sync_lock:
if h in self.pages:
if str(request) != self.pages[h]:
report.add_message('Duplicate of: {0}'.format(self.pages[h]))
dup = True
else:
self.pages[h] = str(request)
if self.content and not dup:
doc = HtmlHelper(response.content)
text = [t for t in doc.get_text(['div', 'p']) if len(t) >= self.content_length]
text.sort(key=lambda k: len(k), reverse=True)
for t in text:
m = hashlib.sha1()
m.update(t.encode())
h = m.hexdigest()
if h in self.paras:
if str(request) != self.paras[h]:
report.add_message(t[0:self.content_length] + '...')
report.add_message('Duplicated from: {0}'.format(self.paras[h]))
else:
self.paras[h] = str(request)
def complete(self, report):
urls = list(self.inbound)
if len(urls) > 0:
urls.sort()
for u in urls:
report.add_message(u)
report.add_message('Total: {0}'.format(len(self.inbound)))
else:
report.add_message('No inbound links found')
def complete(self, report):
urls = list(self.inbound)
if len(urls) > 0:
urls.sort()
for u in urls:
report.add_message(u)
report.add_message('Total: {0}'.format(len(self.inbound)))
else:
report.add_message('No inbound links found')
def process(self, request, response, report):
if request.source == self.name:
err = False
if response.status >= 500:
err = True
report.add_warning('Possible SQL injection')
elif self.xss.search(response.content):
err = True
report.add_warning('Possible XSS')
if 'vector' in request.meta and err:
if request.meta['vector'] == 'post_data':
report.add_message('Post data: {0}'.format(request.post_data))
elif request.meta['vector'] == 'headers':
report.add_message('Request headers: {0}'.format(request.headers))
elif response.is_html and response.status < 500:
# Don't attack error pages - can't tell if it worked without matching against known database error text
doc = HtmlHelper(response.content)
for atk in self.attacks:
if self.quick:
self._inject_all(request, doc, atk)
else:
self._inject_each(request, doc, atk)
def process(self, request, response, report):
global _tidy_available
#TODO: Hash errors and don't log duplicate error sets (just a reference)
if response.is_html and _tidy_available:
try:
doc, err = tidy_document(response.content, options=self.options)
except:
report.add_message('Error parsing: [{0}]'.format(str(request)))
else:
c = 0
for e in err.splitlines():
if self._log(e):
c += 1
report.add_message('{0}'.format(re.sub('^line\\b', 'Line', e)))
if c > 0:
report.add_message('Total: {0}'.format(c))
def process(self, request, response, report):
for rx in self.expressions.items():
inv_h = inv_b = False
if rx[0][0] == '^':
inv_h = True
elif rx[0][0] == '_':
inv_b = True
if inv_h:
if not rx[1].search(str(response.headers)):
report.add_message('Filter: [{0}] not found in headers'.format(rx[0]))
elif not inv_b:
mtchs = rx[1].finditer(str(response.headers))
for mtch in mtchs:
report.add_message('Filter: [{0}] found: [{1}] in headers'.format(rx[0], mtch.group()))
if response.is_html:
if inv_b:
if not rx[1].search(str(response.content)):
report.add_message('Filter: [{0}] not found'.format(rx[0]))
elif not inv_h:
mtchs = rx[1].finditer(response.content)
for mtch in mtchs:
report.add_message('Filter: [{0}] found: [{1}]'.format(rx[0], mtch.group()))
def process(self, request, response, report):
if response.is_html:
doc = HtmlHelper(response.content)
missing = []
empty = []
multiple = []
title = [t for t in doc.get_elements('title')]
if len(title) == 0:
missing.append('title')
elif len(title) > 1:
multiple.append('title')
else:
txt = [t for t in title[0].get_text()]
if len(txt) == 0:
empty.append('title')
meta = {'description': [0, ''], 'keywords': [0, '']}
for e in doc.get_elements('meta'):
names = [n for n in e.get_attribute('name')]
if len(names) > 0:
name = names[0][2].lower()
if name in meta:
meta[name][0] += 1
content = [c for c in e.get_attribute('content')]
if len(content[0][2]) > 0:
meta[name][1] = content[0][2]
for m in meta:
if meta[m][0] == 0:
missing.append(m)
elif meta[m][0] > 1:
multiple.append(m)
elif len(meta[m][1]) == 0:
empty.append(m)
if len(missing) > 0:
report.add_message('Missing: {0}'.format(str(missing)))
if len(empty) > 0:
report.add_message('Empty: {0}'.format(str(empty)))
if len(multiple) > 0:
report.add_message('Multiple: {0}'.format(str(multiple)))
def process(self, request, response, report):
if response.is_html:
doc = HtmlHelper(response.content)
missing = []
empty = []
multiple = []
title = [t for t in doc.get_elements('title')]
if len(title) == 0:
missing.append('title')
elif len(title) > 1:
multiple.append('title')
else:
txt = [t for t in title[0].get_text()]
if len(txt) == 0:
empty.append('title')
meta = {'description': [0, ''], 'keywords': [0, '']}
for e in doc.get_elements('meta'):
names = [n for n in e.get_attribute('name')]
if len(names) > 0:
name = names[0][2].lower()
if name in meta:
meta[name][0] += 1
content = [c for c in e.get_attribute('content')]
if len(content[0][2]) > 0:
meta[name][1] = content[0][2]
for m in meta:
if meta[m][0] == 0:
missing.append(m)
elif meta[m][0] > 1:
multiple.append(m)
elif len(meta[m][1]) == 0:
empty.append(m)
if len(missing) > 0:
report.add_message('Missing: {0}'.format(str(missing)))
if len(empty) > 0:
report.add_message('Empty: {0}'.format(str(empty)))
if len(multiple) > 0:
report.add_message('Multiple: {0}'.format(str(multiple)))
def process(self, request, response, report):
global _tidy_available
#TODO: Hash errors and don't log duplicate error sets (just a reference)
if response.is_html and _tidy_available:
try:
doc, err = tidy_document(response.content,
options=self.options)
except:
report.add_message('Error parsing: [{0}]'.format(str(request)))
else:
c = 0
for e in err.splitlines():
if self._log(e):
c += 1
report.add_message('{0}'.format(
re.sub('^line\\b', 'Line', e)))
if c > 0:
report.add_message('Total: {0}'.format(c))
def process(self, request, response, report):
if response.is_html and response.status < 300:
m = hashlib.sha1()
m.update(response.content.encode())
h = m.hexdigest()
dup = False
with self.sync_lock:
if h in self.pages:
if str(request) != self.pages[h]:
report.add_message('Duplicate of: {0}'.format(
self.pages[h]))
dup = True
else:
self.pages[h] = str(request)
if self.content and not dup:
doc = HtmlHelper(response.content)
text = [
t for t in doc.get_text(['div', 'p'])
if len(t) >= self.content_length
]
text.sort(key=lambda k: len(k), reverse=True)
for t in text:
m = hashlib.sha1()
m.update(t.encode())
h = m.hexdigest()
if h in self.paras:
if str(request) != self.paras[h]:
report.add_message(t[0:self.content_length] +
'...')
report.add_message(
'Duplicated from: {0}'.format(
self.paras[h]))
else:
self.paras[h] = str(request)
def _check(self, domain, report):
today = datetime.date.today()
try:
d = DomainInfo(domain)
except gaierror:
report.add_warning('Domain not found: {0}'.format(domain))
return
if not domain == self.main_domain:
url = 'http://www.{0}/'.format(domain)
req = self._create_request(url, url)
req.modules = [self]
self.sitecheck.request_queue.put(req)
report.add_message('Nameservers:')
for ns in d.name_servers:
report.add_message('\t{0}'.format(ns))
if d.zone_transfer:
report.add_message('Zone Transfer Permitted')
if type(d.domain_expiry) == datetime.date:
rem = (d.domain_expiry - today).days
if rem < 0:
report.add_message('Domain expired {0}'.format(d.domain_expiry))
else:
report.add_message('Domain expires in {0} days'.format(rem))
elif d.domain_expiry:
report.add_message('Domain expires on: {0}'.format(d.domain_expiry))
else:
report.add_warning('Unable to determine domain expiry date')
if d.spf:
report.add_message('SPF: {0}'.format(d.spf))
else:
report.add_warning('No SPF record found')
report.add_message('Hosts:')
for host in d.hosts:
h = d.hosts[host]
report.add_message('\t{0}'.format(h.address))
if h.name:
report.add_message('\t\tReverse DNS: {0}'.format(h.name))
else:
report.add_warning('\t\t No reverse DNS')
report.add_message('\t\tRecords: {0}'.format(', '.join(h.records)))
if h.cert_expiry:
rem = (h.cert_expiry - today).days
if rem < 0:
report.add_message('\t\tCertificate expired {0}'.format(h.cert_expiry))
else:
report.add_message('\t\tCertificate expires in {0} days'.format(rem))
if h.sslv2:
report.add_warning('\t\tInsecure ciphers supported')
if self.relay:
relay, failed = test_relay(h.address, port=25)
if relay:
for f in failed:
report.add_warning('\t\tPossible open relay (port 25): {0} -> {1}'.format(f[0], f[1]))
relay, failed = test_relay(h.address, port=587)
if relay:
for f in failed:
report.add_warning('\t\tPossible open relay (port 587): {0} -> {1}'.format(f[0], f[1]))
def complete(self, report):
if self.count > 0:
report.add_message('\nSummary: Min {:.2f}, Max {:.2f}, Avg {:.2f}'.format(self.min, self.max, self.total / self.count))
def process(self, request, response, report):
if response.status >= 400:
report.add_message('Status: [{0} {1}]'.format(
response.status, response.message))
if request.referrer and len(request.referrer) > 0:
report.add_message('Referrer: [{0}]'.format(request.referrer))
def complete(self, report):
if self.count > 0:
report.add_message(
'\nSummary: Min {:.2f}, Max {:.2f}, Avg {:.2f}'.format(
self.min, self.max, self.total / self.count))
def _check(self, domain, report):
today = datetime.date.today()
try:
d = DomainInfo(domain)
except gaierror:
report.add_warning('Domain not found: {0}'.format(domain))
return
if not domain == self.main_domain:
url = 'http://www.{0}/'.format(domain)
req = self._create_request(url, url)
req.modules = [self]
self.sitecheck.request_queue.put(req)
report.add_message('Nameservers:')
for ns in d.name_servers:
report.add_message('\t{0}'.format(ns))
if d.zone_transfer:
report.add_message('Zone Transfer Permitted')
if type(d.domain_expiry) == datetime.date:
rem = (d.domain_expiry - today).days
if rem < 0:
report.add_message('Domain expired {0}'.format(
d.domain_expiry))
else:
report.add_message('Domain expires in {0} days'.format(rem))
elif d.domain_expiry:
report.add_message('Domain expires on: {0}'.format(
d.domain_expiry))
else:
report.add_warning('Unable to determine domain expiry date')
if d.spf:
report.add_message('SPF: {0}'.format(d.spf))
else:
report.add_warning('No SPF record found')
report.add_message('Hosts:')
for host in d.hosts:
h = d.hosts[host]
report.add_message('\t{0}'.format(h.address))
if h.name:
report.add_message('\t\tReverse DNS: {0}'.format(h.name))
else:
report.add_warning('\t\t No reverse DNS')
report.add_message('\t\tRecords: {0}'.format(', '.join(h.records)))
if h.cert_expiry:
rem = (h.cert_expiry - today).days
if rem < 0:
report.add_message('\t\tCertificate expired {0}'.format(
h.cert_expiry))
else:
report.add_message(
'\t\tCertificate expires in {0} days'.format(rem))
if h.sslv2:
report.add_warning('\t\tInsecure ciphers supported')
if self.relay:
relay, failed = test_relay(h.address, port=25)
if relay:
for f in failed:
report.add_warning(
'\t\tPossible open relay (port 25): {0} -> {1}'.
format(f[0], f[1]))
relay, failed = test_relay(h.address, port=587)
if relay:
for f in failed:
report.add_warning(
'\t\tPossible open relay (port 587): {0} -> {1}'.
format(f[0], f[1]))
def complete(self, report):
if len(self.pages) > 0:
report.add_message('{0}/{1} pages unmatched\n'.format(len(self.pages), self.total))
for p in self.pages:
report.add_message(p)
def complete(self, report):
if len(self.pages) > 0:
report.add_message('{0}/{1} pages unmatched\n'.format(
len(self.pages), self.total))
for p in self.pages:
report.add_message(p)
def process(self, request, response, report):
if response.status >= 400:
report.add_message('Status: [{0} {1}]'.format(response.status, response.message))
if request.referrer and len(request.referrer) > 0:
report.add_message('Referrer: [{0}]'.format(request.referrer))