def test_overriding_port(self):
x = webpage.Webpage('http://test.ing-site.com:81', {}, {}, '80')
request = x.request()
self.assertEqual(request['host'], 'test.ing-site.com')
self.assertEqual(request['port'], '80')
response = x.response()
self.assertEqual(response['status_code'], '200')
def test_custom_parameter_port(self):
x = webpage.Webpage('http://test.ing-site.com', {}, {}, '81')
request = x.request()
self.assertEqual(request['host'], 'test.ing-site.com')
self.assertEqual(request['port'], '81')
response = x.response()
self.assertEqual(response['status_code'], '200')
def test_get_parameters(self):
x = webpage.Webpage('test.ing-site.com', {'key':'value'}, {}, '', '',
'GET')
response = x.response()
self.assertEqual(response['status_code'], '200')
self.assertEqual(response['body'], 'Webpage:<br>GET: key=value')
request = x.request()
self.assertEqual(request['method'], 'GET')
def test_invalid_protocol(self):
x = webpage.Webpage('test.ing-site.com', {}, {}, '', 'muchmuchwrong')
request = x.request()
self.assertEqual(request['host'], 'test.ing-site.com')
self.assertEqual(request['protocol'], 'http')
self.assertEqual(request['port'], '80')
response = x.response()
self.assertEqual(response['status_code'], '200')
def test_two_one_needed_redirection(self):
x = webpage.Webpage('test.ing-site.com?redirect=1', {}, {}, '', '', '',
2)
response = x.response()
self.assertEqual(response['status_code'], '200')
redirects = x.redirects()
self.assertEqual(redirects['follow'], 2)
self.assertEqual(redirects['used'], 1)
def test_protocol(self):
x = webpage.Webpage('https://test.ing-site.com')
request = x.request()
self.assertEqual(request['host'], 'test.ing-site.com')
self.assertEqual(request['protocol'], 'https')
self.assertEqual(request['port'], '443')
response = x.response()
self.assertEqual(response['status_code'], '200')
def test_one_redirection(self):
x = webpage.Webpage('test.ing-site.com?redirect=2', {}, {}, '', '', '',
1)
response = x.response()
self.assertEqual(response['status_code'], '301')
redirects = x.redirects()
self.assertEqual(redirects['follow'], 1)
self.assertEqual(redirects['used'], 1)
def test_post_parameters_url(self):
x = webpage.Webpage('test.ing-site.com?key=value', {}, {}, '', '',
'POST')
response = x.response()
self.assertEqual(response['status_code'], '200')
self.assertEqual(response['body'], 'Webpage:<br>POST: key=value')
request = x.request()
self.assertEqual(request['method'], 'POST')
def test_saved_file(self):
x = webpage.Webpage('test.ing-site.com', {'key':'value'}, {}, '', '',
'POST', 0, 0, 0, '/tmp/index.txt')
response = x.response()
self.assertEqual(response['status_code'], '200')
self.assertEqual(response['body'], 'Webpage:<br>POST: key=value')
f = open('/tmp/index.txt', 'r')
file_content = f.read()
self.assertEqual(file_content, 'Webpage:<br>POST: key=value')
f.close()
def test_encode_parameters(self):
x = webpage.Webpage('test.ing-site.com', {'key':'!"#$%&\'()*+,-./01234'+
'56789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrst'+
'uvwxyz{|}~'})
request = x.request()
self.assertEqual(request['parameter_string'], 'key=%21%22%23%24%25%26'+
'%27%28%29%2A%2B%2C-.%2F0123456789%3A%3B%3C%3D%3E%3F%40ABCDEFGHIJK'+
'LMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C'+
'%7D%7E')
response = x.response()
self.assertEqual(response['status_code'], '200')
self.assertEqual(response['body'], 'Webpage:<br>GET: key=!"#$%&\'()*+,'+
'-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijkl'+
'mnopqrstuvwxyz{|}~')
def test_basic_webpage(self):
x = webpage.Webpage('test.ing-site.com')
request = x.request()
self.assertEqual(request['headers'], {})
self.assertEqual(request['host'], 'test.ing-site.com')
self.assertEqual(request['parameter_string'], '')
self.assertEqual(request['protocol'], 'http')
self.assertEqual(request['location'], '/')
self.assertEqual(request['parameters'], {})
self.assertEqual(request['url'], 'test.ing-site.com')
self.assertEqual(request['parsed_url'], 'test.ing-site.com:80')
self.assertEqual(request['port'], '80')
self.assertEqual(request['method'], 'GET')
response = x.response()
self.assertEqual(response['status_code'], '200')
def __init__(self, datafile, goalsfile, anchortxt, backaction, verbose, maxlen, embedding):
# this method returns simulator, state/action vocabularies, and the maximum number of actions
import pickle
import webpage
self.query_dest_id = pickle.load(open(goalsfile, 'r'))#'data/uiucgoal.pickle'
self.num_goals = len(self.query_dest_id ) * 2
self.wk = webpage.Webpage(datafile)#'data/uiucdataset.hdf5'
self.wk.set_cleaned(True)
self.max_actions, self.min_actions, self.avg_actions = self.wk.get_max_min_avg_nactions()
self.nstates = len(self.wk.get_titles_pos())
print('action statistics ', self.max_actions, self.min_actions, self.avg_actions)
print(' number of states',self.nstates)
self.anchortxt = anchortxt
self.backaction = backaction
self.verbose = verbose
self.maxlen = maxlen
self.embedding = embedding
self.dict_wordId = self.dict_actionId = None
self.reset()
def test_valid_SSL(self):
x = webpage.Webpage('https://encrypted.google.com', {}, {}, '', '', '',
0, 0, 1)
response = x.response()
self.assertEqual(response['status_code'], '200')