Exemple #1
0
	def process(self, request, response, report):
		global _enchant_available
		if response.is_html and _enchant_available:
			doc = HtmlHelper(response.content)
			doc.strip_comments()
			doc.strip_elements(('script', 'style'))

			words = {}
			with self.sync_lock:
				for txt in doc.get_text():
					self._check(txt, words)
				for txt in doc.get_attribute('title'):
					self._check(txt[2], words)
				for txt in doc.get_attribute('alt'):
					self._check(txt[2], words)
				for e in doc.get_elements('meta'):
					names = [n for n in e.get_attribute('name')]
					if len(names) > 0:
						name = names[0][2].lower()
						if name == 'description' or name == 'keywords':
							content = [c for c in e.get_attribute('content')]
							if len(content) > 0:
								self._check(content[0][2], words)

			if len(words) > 0:
				keys = list(words.keys())
				keys.sort()
				for k in keys:
					report.add_message('Word: [{0}] x {1} ({2})'.format(words[k][0], words[k][1], words[k][2]))
Exemple #2
0
	def process(self, request, response, report):
		if response.is_html:
			doc = HtmlHelper(response.content)
			doc.strip_comments()
			doc.strip_elements(('script', 'style'))

			th = TextHelper()
			for txt in doc.get_text():
				th.append(txt)

			if len(th) > 0:
				twrd = float(th.word_count())
				tsnt = float(th.sentence_count())
				tsyl = float(th.syllable_count())

				fkre = 206.835 - 1.015 * (twrd / tsnt) - 84.6 * (tsyl / twrd)

				with self.sync_lock:
					self.count += 1
					self.total += fkre
					if self.min == None:
						self.min = fkre
					else:
						self.min = min(self.min, fkre)

					if self.max == None:
						self.max = fkre
					else:
						self.max = max(self.max, fkre)

				if fkre < self.threshold:
					report.add_message('Readability: [{1:.2f}]'.format(str(request), fkre))
Exemple #3
0
	def begin(self, report):
		if self.spell_checker:
			report.add_message('Language: {0}'.format(self.language))
			if self.dictionary:
				report.add_message('Using custom dictionary [{0}]'.format(self.dictionary))
		else:
			report.add_error('pyenchant not available')
Exemple #4
0
    def process(self, request, response, report):
        if response.is_html:
            doc = HtmlHelper(response.content)
            doc.strip_comments()
            doc.strip_elements(('script', 'style'))

            th = TextHelper()
            for txt in doc.get_text():
                th.append(txt)

            if len(th) > 0:
                twrd = float(th.word_count())
                tsnt = float(th.sentence_count())
                tsyl = float(th.syllable_count())

                fkre = 206.835 - 1.015 * (twrd / tsnt) - 84.6 * (tsyl / twrd)

                with self.sync_lock:
                    self.count += 1
                    self.total += fkre
                    if self.min == None:
                        self.min = fkre
                    else:
                        self.min = min(self.min, fkre)

                    if self.max == None:
                        self.max = fkre
                    else:
                        self.max = max(self.max, fkre)

                if fkre < self.threshold:
                    report.add_message('Readability: [{1:.2f}]'.format(
                        str(request), fkre))
Exemple #5
0
    def process(self, request, response, report):
        global _enchant_available
        if response.is_html and _enchant_available:
            doc = HtmlHelper(response.content)
            doc.strip_comments()
            doc.strip_elements(('script', 'style'))

            words = {}
            with self.sync_lock:
                for txt in doc.get_text():
                    self._check(txt, words)
                for txt in doc.get_attribute('title'):
                    self._check(txt[2], words)
                for txt in doc.get_attribute('alt'):
                    self._check(txt[2], words)
                for e in doc.get_elements('meta'):
                    names = [n for n in e.get_attribute('name')]
                    if len(names) > 0:
                        name = names[0][2].lower()
                        if name == 'description' or name == 'keywords':
                            content = [c for c in e.get_attribute('content')]
                            if len(content) > 0:
                                self._check(content[0][2], words)

            if len(words) > 0:
                keys = list(words.keys())
                keys.sort()
                for k in keys:
                    report.add_message('Word: [{0}] x {1} ({2})'.format(
                        words[k][0], words[k][1], words[k][2]))
Exemple #6
0
    def process(self, request, response, report):
        if request.source == self.name:
            err = False
            if response.status >= 500:
                err = True
                report.add_warning('Possible SQL injection')
            elif self.xss.search(response.content):
                err = True
                report.add_warning('Possible XSS')

            if 'vector' in request.meta and err:
                if request.meta['vector'] == 'post_data':
                    report.add_message('Post data: {0}'.format(
                        request.post_data))
                elif request.meta['vector'] == 'headers':
                    report.add_message('Request headers: {0}'.format(
                        request.headers))

        elif response.is_html and response.status < 500:
            # Don't attack error pages - can't tell if it worked without matching against known database error text
            doc = HtmlHelper(response.content)
            for atk in self.attacks:
                if self.quick:
                    self._inject_all(request, doc, atk)
                else:
                    self._inject_each(request, doc, atk)
Exemple #7
0
 def begin(self, report):
     if self.spell_checker:
         report.add_message('Language: {0}'.format(self.language))
         if self.dictionary:
             report.add_message('Using custom dictionary [{0}]'.format(
                 self.dictionary))
     else:
         report.add_error('pyenchant not available')
Exemple #8
0
 def _report(self, report, urls):
     out = list(urls)
     if len(out) > 0:
         out.sort()
         for url in out:
             if url.count(' ') > 0:
                 report.add_message('-> [{0}] *Unencoded'.format(url))
             else:
                 report.add_message('-> [{0}]'.format(url))
Exemple #9
0
	def _report(self, report, urls):
		out = list(urls)
		if len(out) > 0:
			out.sort()
			for url in out:
				if url.count(' ') > 0:
					report.add_message('-> [{0}] *Unencoded'.format(url))
				else:
					report.add_message('-> [{0}]'.format(url))
Exemple #10
0
	def process(self, request, response, report):
		if response.is_html:
			doc = HtmlHelper(response.content)
			for comment in doc.get_comments():
				c = comment.strip()
				if c.startswith('[if') and c.endswith('<![endif]'):
					# Ignore IE conditional comments
					pass
				else:
					report.add_message('Comment:\t{0}'.format(re.sub('\r?\n', '\n\t\t\t\t', c, re.MULTILINE)))
Exemple #11
0
	def process(self, request, response, report):
		if response.is_html and request.protocol.lower() == 'https':
			doc = HtmlHelper(response.content)

			for e, a, v in doc.get_attribute('src'): #img, script
				if v.lower().startswith('http:'):
					report.add_message('{0}'.format(v))

			for e, a, v in doc.get_attribute('href', 'link'):
				if v.lower().startswith('http:'):
					report.add_message('{0}'.format(v))
Exemple #12
0
 def process(self, request, response, report):
     if response.is_html:
         doc = HtmlHelper(response.content)
         for comment in doc.get_comments():
             c = comment.strip()
             if c.startswith('[if') and c.endswith('<![endif]'):
                 # Ignore IE conditional comments
                 pass
             else:
                 report.add_message('Comment:\t{0}'.format(
                     re.sub('\r?\n', '\n\t\t\t\t', c, re.MULTILINE)))
Exemple #13
0
    def process(self, request, response, report):
        if response.is_html and request.protocol.lower() == 'https':
            doc = HtmlHelper(response.content)

            for e, a, v in doc.get_attribute('src'):  #img, script
                if v.lower().startswith('http:'):
                    report.add_message('{0}'.format(v))

            for e, a, v in doc.get_attribute('href', 'link'):
                if v.lower().startswith('http:'):
                    report.add_message('{0}'.format(v))
Exemple #14
0
    def process(self, request, response, report):
        for rx in self.expressions.items():
            inv_h = inv_b = False
            if rx[0][0] == '^':
                inv_h = True
            elif rx[0][0] == '_':
                inv_b = True

            if inv_h:
                if not rx[1].search(str(response.headers)):
                    report.add_message(
                        'Filter: [{0}] not found in headers'.format(rx[0]))
            elif not inv_b:
                mtchs = rx[1].finditer(str(response.headers))
                for mtch in mtchs:
                    report.add_message(
                        'Filter: [{0}] found: [{1}] in headers'.format(
                            rx[0], mtch.group()))

            if response.is_html:
                if inv_b:
                    if not rx[1].search(str(response.content)):
                        report.add_message('Filter: [{0}] not found'.format(
                            rx[0]))
                elif not inv_h:
                    mtchs = rx[1].finditer(response.content)
                    for mtch in mtchs:
                        report.add_message('Filter: [{0}] found: [{1}]'.format(
                            rx[0], mtch.group()))
Exemple #15
0
	def process(self, request, response, report):
		global _tidy_available
		#TODO: Hash errors and don't log duplicate error sets (just a reference)
		if response.is_html and _tidy_available:
			try:
				doc, err = tidy_document(response.content, options=self.options)
			except:
				report.add_error('Unable to parse response')
			else:
				l = err.splitlines()
				if len(l) > 0:
					for e in l:
						report.add_message('{0}'.format(re.sub('^line\\b', 'Line', e)))

					report.add_message('Total: {0}'.format(len(l)))
Exemple #16
0
    def process(self, request, response, report):
        global _tidy_available
        #TODO: Hash errors and don't log duplicate error sets (just a reference)
        if response.is_html and _tidy_available:
            try:
                doc, err = tidy_document(response.content,
                                         options=self.options)
            except:
                report.add_error('Unable to parse response')
            else:
                l = err.splitlines()
                if len(l) > 0:
                    for e in l:
                        report.add_message('{0}'.format(
                            re.sub('^line\\b', 'Line', e)))

                    report.add_message('Total: {0}'.format(len(l)))
Exemple #17
0
	def process(self, request, response, report):
		if response.is_html and response.status < 300:
			m = hashlib.sha1()
			m.update(response.content.encode())
			h = m.hexdigest()

			dup = False
			with self.sync_lock:
				if h in self.pages:
					if str(request) != self.pages[h]:
						report.add_message('Duplicate of: {0}'.format(self.pages[h]))
						dup = True
				else:
					self.pages[h] = str(request)

				if self.content and not dup:
					doc = HtmlHelper(response.content)
					text = [t for t in doc.get_text(['div', 'p']) if len(t) >= self.content_length]
					text.sort(key=lambda k: len(k), reverse=True)
					for t in text:
						m = hashlib.sha1()
						m.update(t.encode())
						h = m.hexdigest()

						if h in self.paras:
							if str(request) != self.paras[h]:
								report.add_message(t[0:self.content_length] + '...')
								report.add_message('Duplicated from: {0}'.format(self.paras[h]))
						else:
							self.paras[h] = str(request)
Exemple #18
0
	def complete(self, report):
		urls = list(self.inbound)
		if len(urls) > 0:
			urls.sort()
			for u in urls:
				report.add_message(u)
			report.add_message('Total: {0}'.format(len(self.inbound)))
		else:
			report.add_message('No inbound links found')
Exemple #19
0
 def complete(self, report):
     urls = list(self.inbound)
     if len(urls) > 0:
         urls.sort()
         for u in urls:
             report.add_message(u)
         report.add_message('Total: {0}'.format(len(self.inbound)))
     else:
         report.add_message('No inbound links found')
Exemple #20
0
	def process(self, request, response, report):
		if request.source == self.name:
			err = False
			if response.status >= 500:
				err = True
				report.add_warning('Possible SQL injection')
			elif self.xss.search(response.content):
				err = True
				report.add_warning('Possible XSS')

			if 'vector' in request.meta and err:
				if request.meta['vector'] == 'post_data':
					report.add_message('Post data: {0}'.format(request.post_data))
				elif request.meta['vector'] == 'headers':
					report.add_message('Request headers: {0}'.format(request.headers))

		elif response.is_html and response.status < 500:
			# Don't attack error pages - can't tell if it worked without matching against known database error text
			doc = HtmlHelper(response.content)
			for atk in self.attacks:
				if self.quick:
					self._inject_all(request, doc, atk)
				else:
					self._inject_each(request, doc, atk)
Exemple #21
0
	def process(self, request, response, report):
		global _tidy_available
		#TODO: Hash errors and don't log duplicate error sets (just a reference)
		if response.is_html and _tidy_available:
			try:
				doc, err = tidy_document(response.content, options=self.options)
			except:
				report.add_message('Error parsing: [{0}]'.format(str(request)))
			else:
				c = 0
				for e in err.splitlines():
					if self._log(e):
						c += 1
						report.add_message('{0}'.format(re.sub('^line\\b', 'Line', e)))

				if c > 0:
					report.add_message('Total: {0}'.format(c))
Exemple #22
0
	def process(self, request, response, report):
		for rx in self.expressions.items():
			inv_h = inv_b = False
			if rx[0][0] == '^':
				inv_h = True
			elif rx[0][0] == '_':
				inv_b = True

			if inv_h:
				if not rx[1].search(str(response.headers)):
					report.add_message('Filter: [{0}] not found in headers'.format(rx[0]))
			elif not inv_b:
				mtchs = rx[1].finditer(str(response.headers))
				for mtch in mtchs:
					report.add_message('Filter: [{0}] found: [{1}] in headers'.format(rx[0], mtch.group()))

			if response.is_html:
				if inv_b:
					if not rx[1].search(str(response.content)):
						report.add_message('Filter: [{0}] not found'.format(rx[0]))
				elif not inv_h:
					mtchs = rx[1].finditer(response.content)
					for mtch in mtchs:
						report.add_message('Filter: [{0}] found: [{1}]'.format(rx[0], mtch.group()))
Exemple #23
0
    def process(self, request, response, report):
        if response.is_html:
            doc = HtmlHelper(response.content)
            missing = []
            empty = []
            multiple = []

            title = [t for t in doc.get_elements('title')]
            if len(title) == 0:
                missing.append('title')
            elif len(title) > 1:
                multiple.append('title')
            else:
                txt = [t for t in title[0].get_text()]
                if len(txt) == 0:
                    empty.append('title')

            meta = {'description': [0, ''], 'keywords': [0, '']}
            for e in doc.get_elements('meta'):
                names = [n for n in e.get_attribute('name')]
                if len(names) > 0:
                    name = names[0][2].lower()
                    if name in meta:
                        meta[name][0] += 1
                        content = [c for c in e.get_attribute('content')]
                        if len(content[0][2]) > 0:
                            meta[name][1] = content[0][2]

            for m in meta:
                if meta[m][0] == 0:
                    missing.append(m)
                elif meta[m][0] > 1:
                    multiple.append(m)
                elif len(meta[m][1]) == 0:
                    empty.append(m)

            if len(missing) > 0:
                report.add_message('Missing: {0}'.format(str(missing)))

            if len(empty) > 0:
                report.add_message('Empty: {0}'.format(str(empty)))

            if len(multiple) > 0:
                report.add_message('Multiple: {0}'.format(str(multiple)))
Exemple #24
0
	def process(self, request, response, report):
		if response.is_html:
			doc = HtmlHelper(response.content)
			missing = []
			empty = []
			multiple = []

			title = [t for t in doc.get_elements('title')]
			if len(title) == 0:
				missing.append('title')
			elif len(title) > 1:
				multiple.append('title')
			else:
				txt = [t for t in title[0].get_text()]
				if len(txt) == 0:
					empty.append('title')

			meta = {'description': [0, ''], 'keywords': [0, '']}
			for e in doc.get_elements('meta'):
				names = [n for n in e.get_attribute('name')]
				if len(names) > 0:
					name = names[0][2].lower()
					if name in meta:
						meta[name][0] += 1
						content = [c for c in e.get_attribute('content')]
						if len(content[0][2]) > 0:
							meta[name][1] = content[0][2]

			for m in meta:
				if meta[m][0] == 0:
					missing.append(m)
				elif meta[m][0] > 1:
					multiple.append(m)
				elif len(meta[m][1]) == 0:
					empty.append(m)

			if len(missing) > 0:
				report.add_message('Missing: {0}'.format(str(missing)))

			if len(empty) > 0:
				report.add_message('Empty: {0}'.format(str(empty)))

			if len(multiple) > 0:
				report.add_message('Multiple: {0}'.format(str(multiple)))
Exemple #25
0
    def process(self, request, response, report):
        global _tidy_available
        #TODO: Hash errors and don't log duplicate error sets (just a reference)
        if response.is_html and _tidy_available:
            try:
                doc, err = tidy_document(response.content,
                                         options=self.options)
            except:
                report.add_message('Error parsing: [{0}]'.format(str(request)))
            else:
                c = 0
                for e in err.splitlines():
                    if self._log(e):
                        c += 1
                        report.add_message('{0}'.format(
                            re.sub('^line\\b', 'Line', e)))

                if c > 0:
                    report.add_message('Total: {0}'.format(c))
Exemple #26
0
    def process(self, request, response, report):
        if response.is_html and response.status < 300:
            m = hashlib.sha1()
            m.update(response.content.encode())
            h = m.hexdigest()

            dup = False
            with self.sync_lock:
                if h in self.pages:
                    if str(request) != self.pages[h]:
                        report.add_message('Duplicate of: {0}'.format(
                            self.pages[h]))
                        dup = True
                else:
                    self.pages[h] = str(request)

                if self.content and not dup:
                    doc = HtmlHelper(response.content)
                    text = [
                        t for t in doc.get_text(['div', 'p'])
                        if len(t) >= self.content_length
                    ]
                    text.sort(key=lambda k: len(k), reverse=True)
                    for t in text:
                        m = hashlib.sha1()
                        m.update(t.encode())
                        h = m.hexdigest()

                        if h in self.paras:
                            if str(request) != self.paras[h]:
                                report.add_message(t[0:self.content_length] +
                                                   '...')
                                report.add_message(
                                    'Duplicated from: {0}'.format(
                                        self.paras[h]))
                        else:
                            self.paras[h] = str(request)
Exemple #27
0
	def _check(self, domain, report):
		today = datetime.date.today()

		try:
			d = DomainInfo(domain)
		except gaierror:
			report.add_warning('Domain not found: {0}'.format(domain))
			return

		if not domain == self.main_domain:
			url = 'http://www.{0}/'.format(domain)
			req = self._create_request(url, url)
			req.modules = [self]
			self.sitecheck.request_queue.put(req)

		report.add_message('Nameservers:')
		for ns in d.name_servers:
			report.add_message('\t{0}'.format(ns))

		if d.zone_transfer:
			report.add_message('Zone Transfer Permitted')

		if type(d.domain_expiry) == datetime.date:
			rem = (d.domain_expiry - today).days
			if rem < 0:
				report.add_message('Domain expired {0}'.format(d.domain_expiry))
			else:
				report.add_message('Domain expires in {0} days'.format(rem))
		elif d.domain_expiry:
			report.add_message('Domain expires on: {0}'.format(d.domain_expiry))
		else:
			report.add_warning('Unable to determine domain expiry date')

		if d.spf:
			report.add_message('SPF: {0}'.format(d.spf))
		else:
			report.add_warning('No SPF record found')

		report.add_message('Hosts:')
		for host in d.hosts:
			h = d.hosts[host]

			report.add_message('\t{0}'.format(h.address))

			if h.name:
				report.add_message('\t\tReverse DNS: {0}'.format(h.name))
			else:
				report.add_warning('\t\t No reverse DNS')

			report.add_message('\t\tRecords: {0}'.format(', '.join(h.records)))

			if h.cert_expiry:
				rem = (h.cert_expiry - today).days
				if rem < 0:
					report.add_message('\t\tCertificate expired {0}'.format(h.cert_expiry))
				else:
					report.add_message('\t\tCertificate expires in {0} days'.format(rem))

			if h.sslv2:
				report.add_warning('\t\tInsecure ciphers supported')

			if self.relay:
				relay, failed = test_relay(h.address, port=25)
				if relay:
					for f in failed:
						report.add_warning('\t\tPossible open relay (port 25): {0} -> {1}'.format(f[0], f[1]))

				relay, failed = test_relay(h.address, port=587)
				if relay:
					for f in failed:
						report.add_warning('\t\tPossible open relay (port 587): {0} -> {1}'.format(f[0], f[1]))
Exemple #28
0
	def complete(self, report):
		if self.count > 0:
			report.add_message('\nSummary: Min {:.2f}, Max {:.2f}, Avg {:.2f}'.format(self.min, self.max, self.total / self.count))
Exemple #29
0
 def process(self, request, response, report):
     if response.status >= 400:
         report.add_message('Status: [{0} {1}]'.format(
             response.status, response.message))
         if request.referrer and len(request.referrer) > 0:
             report.add_message('Referrer: [{0}]'.format(request.referrer))
Exemple #30
0
 def complete(self, report):
     if self.count > 0:
         report.add_message(
             '\nSummary: Min {:.2f}, Max {:.2f}, Avg {:.2f}'.format(
                 self.min, self.max, self.total / self.count))
Exemple #31
0
    def _check(self, domain, report):
        today = datetime.date.today()

        try:
            d = DomainInfo(domain)
        except gaierror:
            report.add_warning('Domain not found: {0}'.format(domain))
            return

        if not domain == self.main_domain:
            url = 'http://www.{0}/'.format(domain)
            req = self._create_request(url, url)
            req.modules = [self]
            self.sitecheck.request_queue.put(req)

        report.add_message('Nameservers:')
        for ns in d.name_servers:
            report.add_message('\t{0}'.format(ns))

        if d.zone_transfer:
            report.add_message('Zone Transfer Permitted')

        if type(d.domain_expiry) == datetime.date:
            rem = (d.domain_expiry - today).days
            if rem < 0:
                report.add_message('Domain expired {0}'.format(
                    d.domain_expiry))
            else:
                report.add_message('Domain expires in {0} days'.format(rem))
        elif d.domain_expiry:
            report.add_message('Domain expires on: {0}'.format(
                d.domain_expiry))
        else:
            report.add_warning('Unable to determine domain expiry date')

        if d.spf:
            report.add_message('SPF: {0}'.format(d.spf))
        else:
            report.add_warning('No SPF record found')

        report.add_message('Hosts:')
        for host in d.hosts:
            h = d.hosts[host]

            report.add_message('\t{0}'.format(h.address))

            if h.name:
                report.add_message('\t\tReverse DNS: {0}'.format(h.name))
            else:
                report.add_warning('\t\t No reverse DNS')

            report.add_message('\t\tRecords: {0}'.format(', '.join(h.records)))

            if h.cert_expiry:
                rem = (h.cert_expiry - today).days
                if rem < 0:
                    report.add_message('\t\tCertificate expired {0}'.format(
                        h.cert_expiry))
                else:
                    report.add_message(
                        '\t\tCertificate expires in {0} days'.format(rem))

            if h.sslv2:
                report.add_warning('\t\tInsecure ciphers supported')

            if self.relay:
                relay, failed = test_relay(h.address, port=25)
                if relay:
                    for f in failed:
                        report.add_warning(
                            '\t\tPossible open relay (port 25): {0} -> {1}'.
                            format(f[0], f[1]))

                relay, failed = test_relay(h.address, port=587)
                if relay:
                    for f in failed:
                        report.add_warning(
                            '\t\tPossible open relay (port 587): {0} -> {1}'.
                            format(f[0], f[1]))
Exemple #32
0
	def complete(self, report):
		if len(self.pages) > 0:
			report.add_message('{0}/{1} pages unmatched\n'.format(len(self.pages), self.total))
			for p in self.pages:
				report.add_message(p)
Exemple #33
0
 def complete(self, report):
     if len(self.pages) > 0:
         report.add_message('{0}/{1} pages unmatched\n'.format(
             len(self.pages), self.total))
         for p in self.pages:
             report.add_message(p)
Exemple #34
0
	def process(self, request, response, report):
		if response.status >= 400:
			report.add_message('Status: [{0} {1}]'.format(response.status, response.message))
			if request.referrer and len(request.referrer) > 0:
				report.add_message('Referrer: [{0}]'.format(request.referrer))