def test_filtering_via_fuzzy_matching(self):
test_name = sys._getframe().f_code.co_name.replace('test_', '')
headers = {}
headers["Host"] = [randomStr(10) + self.input + randomStr(10)]
response = yield self.doRequest(self.localOptions['backend'],
headers=headers)
self.check_for_censorship(response.body, test_name)
def test_filtering_via_fuzzy_matching(self):
test_name = sys._getframe().f_code.co_name.replace('test_', '')
headers = {}
headers["Host"] = [randomStr(10) + self.input + randomStr(10)]
response = yield self.doRequest(self.localOptions['backend'],
headers=headers)
self.check_for_censorship(response.body, test_name)
def test_fuzzy_match_blocking(self):
hostname_field = randomStr(10) + self.input + randomStr(10)
payload = "GET / HTTP/1.1\n\r"
payload += "Host: %s\n\r" % hostname_field
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def test_fuzzy_match_blocking(self):
hostname_field = randomStr(10) + self.input + randomStr(10)
payload = "GET / HTTP/1.1\n\r"
payload += "Host: %s\n\r" % hostname_field
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def test_random_invalid_request(self):
"""
We test sending data to a TCP echo server, if what we get back is not
what we have sent then there is tampering going on.
This is for example what squid will return when performing such
request:
HTTP/1.0 400 Bad Request
Server: squid/2.6.STABLE21
Date: Sat, 23 Jul 2011 02:22:44 GMT
Content-Type: text/html
Content-Length: 1178
Expires: Sat, 23 Jul 2011 02:22:44 GMT
X-Squid-Error: ERR_INVALID_REQ 0
X-Cache: MISS from cache_server
X-Cache-Lookup: NONE from cache_server:3128
Via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
Proxy-Connection: close
"""
payload = randomStr(10) + "\n\r"
def got_all_data(received_array):
if not self.localOptions['nopayloadmatch']:
first = received_array[0]
if first != payload:
self.report['tampering'] = True
else:
self.report['tampering'] = 'unknown'
d = self.sendPayload(payload)
d.addCallback(got_all_data)
return d
def setupTestCases(self, test_cases):
"""
Creates all the necessary test_cases (a list of tuples containing the
NetTestCase (test_class, test_method))
example:
[(test_classA, test_method1),
(test_classA, test_method2),
(test_classA, test_method3),
(test_classA, test_method4),
(test_classA, test_method5),
(test_classB, test_method1),
(test_classB, test_method2)]
Note: the inputs must be valid for test_classA and test_classB.
net_test_file:
is either a file path or a file like object that will be used to
generate the test_cases.
"""
test_class, _ = test_cases[0]
self.testVersion = test_class.version
self.testName = test_class_name_to_name(test_class.name)
self.testCases = test_cases
self.testClasses = set([])
self.testHelpers = {}
if config.reports.unique_id is True:
self.reportID = randomStr(64)
for test_class, test_method in self.testCases:
self.testClasses.add(test_class)
def get_headers(self):
headers = {}
if self.localOptions['headers']:
try:
f = open(self.localOptions['headers'])
except IOError:
raise Exception("Specified input file does not exist")
content = ''.join(f.readlines())
f.close()
headers = yaml.safe_load(content)
return headers
else:
# XXX generate these from a random choice taken from
# whatheaders.com
# http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip
headers = {
"User-Agent": [
random.choice(
net.userAgents)],
"Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
"Accept-Encoding": ["gzip,deflate,sdch"],
"Accept-Language": ["en-US,en;q=0.8"],
"Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"],
"Host": [
randomStr(15) +
'.com']}
return headers
def test_random_invalid_request(self):
"""
We test sending data to a TCP echo server, if what we get back is not
what we have sent then there is tampering going on.
This is for example what squid will return when performing such
request:
HTTP/1.0 400 Bad Request
Server: squid/2.6.STABLE21
Date: Sat, 23 Jul 2011 02:22:44 GMT
Content-Type: text/html
Content-Length: 1178
Expires: Sat, 23 Jul 2011 02:22:44 GMT
X-Squid-Error: ERR_INVALID_REQ 0
X-Cache: MISS from cache_server
X-Cache-Lookup: NONE from cache_server:3128
Via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
Proxy-Connection: close
"""
payload = randomStr(10) + "\n\r"
def got_all_data(received_array):
if not self.localOptions['nopayloadmatch']:
first = received_array[0]
if first != payload:
self.report['tampering'] = True
else:
self.report['tampering'] = 'unknown'
d = self.sendPayload(payload)
d.addCallback(got_all_data)
return d
def test_subdomain_blocking(self):
payload = "GET / HTTP/1.1\n\r"
payload += "Host: %s\n\r" % randomStr(10) + '.' + self.input
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def setupTestCases(self, test_cases):
"""
Creates all the necessary test_cases (a list of tuples containing the
NetTestCase (test_class, test_method))
example:
[(test_classA, test_method1),
(test_classA, test_method2),
(test_classA, test_method3),
(test_classA, test_method4),
(test_classA, test_method5),
(test_classB, test_method1),
(test_classB, test_method2)]
Note: the inputs must be valid for test_classA and test_classB.
net_test_file:
is either a file path or a file like object that will be used to
generate the test_cases.
"""
test_class, _ = test_cases[0]
self.testVersion = test_class.version
self.testName = test_class_name_to_name(test_class.name)
self.testCases = test_cases
self.testClasses = set([])
self.testHelpers = {}
if config.reports.unique_id is True:
self.reportID = randomStr(64)
for test_class, test_method in self.testCases:
self.testClasses.add(test_class)
def get_headers(self):
headers = {}
if self.localOptions['headers']:
try:
f = open(self.localOptions['headers'])
except IOError:
raise Exception("Specified input file does not exist")
content = ''.join(f.readlines())
f.close()
headers = yaml.safe_load(content)
return headers
else:
# XXX generate these from a random choice taken from
# whatheaders.com
# http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip
headers = {
"User-Agent": [
random.choice(
net.userAgents)],
"Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
"Accept-Encoding": ["gzip,deflate,sdch"],
"Accept-Language": ["en-US,en;q=0.8"],
"Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"],
"Host": [
randomStr(15) +
'.com']}
return headers
def _setupTestCases(self, test_cases):
"""
Creates all the necessary test_cases (a list of tuples containing the
NetTestCase (test_class, test_method))
example:
[(test_classA, [test_method1,
test_method2,
test_method3,
test_method4,
test_method5]),
(test_classB, [test_method1,
test_method2])]
Note: the inputs must be valid for test_classA and test_classB.
net_test_file:
is either a file path or a file like object that will be used to
generate the test_cases.
"""
test_class, _ = test_cases[0]
self.testName = normalizeTestName(test_class.name)
self.testVersion = test_class.version
self._testCases = test_cases
self.usageOptions = usageOptionsFactory(self.testName,
self.testVersion)
if config.reports.unique_id is True:
self.reportId = randomStr(64)
for test_class, test_methods in self._testCases:
self._accumulateTestOptions(test_class)
def test_subdomain_blocking(self):
payload = "GET / HTTP/1.1\n\r"
payload += "Host: %s\n\r" % randomStr(10) + "." + self.input
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def generateReportID():
"""
Generates a report ID for usage in the database backed oonib collector.
XXX note how this function is different from the one in report/api.py
"""
report_id = randomStr(100)
return report_id
def generateReportID():
"""
Generates a report ID for usage in the database backed oonib collector.
XXX note how this function is different from the one in report/api.py
"""
report_id = randomStr(100)
return report_id
def test_random_big_request_method(self):
"""
This generates a request that looks like this:
Xx*512 / HTTP/1.1
"""
payload = randomStr(1024) + ' / HTTP/1.1\n\r'
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def test_random_big_request_method(self):
"""
This generates a request that looks like this:
Xx*512 / HTTP/1.1
"""
payload = randomStr(1024) + ' / HTTP/1.1\n\r'
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def test_random_invalid_version_number(self):
"""
This generates a request that looks like this:
GET / HTTP/XxX
"""
payload = 'GET / HTTP/' + randomStr(3)
payload += '\n\r'
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def test_random_invalid_version_number(self):
"""
This generates a request that looks like this:
GET / HTTP/XxX
"""
payload = 'GET / HTTP/' + randomStr(3)
payload += '\n\r'
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def test_random_invalid_field_count(self):
"""
This generates a request that looks like this:
XxXxX XxXxX XxXxX XxXxX
This may trigger some bugs in the HTTP parsers of transparent HTTP
proxies.
"""
payload = ' '.join(randomStr(5) for x in range(4))
payload += "\n\r"
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def test_random_invalid_field_count(self):
"""
This generates a request that looks like this:
XxXxX XxXxX XxXxX XxXxX
This may trigger some bugs in the HTTP parsers of transparent HTTP
proxies.
"""
payload = ' '.join(randomStr(5) for x in range(4))
payload += "\n\r"
d = self.sendPayload(payload)
d.addCallback(self.check_for_manipulation, payload)
return d
def get_headers(self):
headers = {}
if self.localOptions['headers']:
# XXX test this code
try:
f = open(self.localOptions['headers'])
except IOError:
raise Exception("Specified input file does not exist")
content = ''.join(f.readlines())
f.close()
headers = yaml.load(content)
return headers
else:
headers = {"User-Agent": [random.choice(net.userAgents)[0]],
"Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
"Accept-Encoding": ["gzip,deflate,sdch"],
"Accept-Language": ["en-US,en;q=0.8"],
"Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"],
"Host": [randomStr(15)+'.com']
}
return headers
def get_headers(self):
headers = {}
if self.localOptions['headers']:
# XXX test this code
try:
f = open(self.localOptions['headers'])
except IOError:
raise Exception("Specified input file does not exist")
content = ''.join(f.readlines())
f.close()
headers = yaml.load(content)
return headers
else:
headers = {
"User-Agent": [random.choice(net.userAgents)[0]],
"Accept": [
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
],
"Accept-Encoding": ["gzip,deflate,sdch"],
"Accept-Language": ["en-US,en;q=0.8"],
"Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"],
"Host": [randomStr(15) + '.com']
}
return headers
def test_filtering_via_fuzzy_matching(self):
headers = {}
headers["Host"] = [randomStr(10) + self.input + randomStr(10)]
return self.doRequest(self.localOptions['backend'],
headers=headers)
def test_filtering_of_subdomain(self):
headers = {}
headers["Host"] = [randomStr(10) + '.' + self.input]
return self.doRequest(self.localOptions['backend'], headers=headers)
def generateReportID():
return otime.timestamp() + '_' + randomStr(20)
def test_filtering_via_fuzzy_matching(self):
headers = {}
headers["Host"] = [randomStr(10) + self.input + randomStr(10)]
return self.doRequest(self.localOptions['backend'], headers=headers)
def generateReportID():
return otime.timestamp() + '_' + randomStr(20)
raise web.HTTPError(400, "Missing Request Field %s" % e)
print "Parsed this data %s" % report_data
software_name = report_data['software_name']
software_version = report_data['software_version']
test_name = report_data['test_name']
test_version = report_data['test_version']
probe_asn = report_data['probe_asn']
content = report_data['content']
if not probe_asn:
probe_asn = "AS0"
report_id = otime.timestamp() + '_' \
+ probe_asn + '_' \
+ randomStr(50)
# The report filename contains the timestamp of the report plus a
# random nonce
report_filename = os.path.join(config.main.report_dir, report_id)
response = {'backend_version': config.backend_version,
'report_id': report_id
}
self.writeToReport(report_filename,
report_data['content'])
self.write(response)
def writeToReport(self, report_filename, data):
def test_filtering_of_subdomain(self):
headers = {}
headers["Host"] = [randomStr(10) + '.' + self.input]
return self.doRequest(self.localOptions['backend'],
headers=headers)