def test_cookies_jar(self):
random_key = "key_" + uuid.uuid4().get_hex()[:10]
random_value = "value_" + uuid.uuid4().get_hex()
random_key2 = "key_" + uuid.uuid4().get_hex()[:10]
random_value2 = "value_" + uuid.uuid4().get_hex()
cookies = ((random_key, random_value),
(random_key2, random_value2))
cookies_jar = cookielib.CookieJar()
r1 = requests.get(build_url("cookies", "set", random_key, random_value),
cookies=cookies_jar, debug=stdout_debug)
self.assertEquals(r1.cookies[random_key], random_value)
rtmp = requests.get(build_url("cookies", "set", random_key2, random_value2),
cookies=cookies_jar, debug=stdout_debug)
for cookie in cookies_jar:
if cookie.name == random_key:
self.assertEquals(cookie.value, random_value)
r3 = requests.get(build_url('cookies'), cookies=cookies_jar, debug=stdout_debug)
json_response = json.loads(r3.content)
for k, v in cookies:
self.assertEquals(json_response[k], v)
def spamspamspam(url,tries):
for i in range(tries):
if "http" in url:
c = hurl.get(url)
else:
c = hurl.get('http://'+url)
if c.status_code is not "302":
print " Returned a {} status code. Not blocking yet. ".format(c.status_code)
else:
print " Blocking! (Code {}) ".format(c.status_code)
def test_auth_denied(self):
username = "******"
password = "******"
http_auth = (username, password)
r = requests.get(build_url('basic-auth', "username", "password"), auth=http_auth)
self.assertEquals(r.status_code, 401)
def get_translations(word):
if word in translations_cache:
return translations_cache[word]
query = translations_query_template.format(urllib.quote(word))
url = "{}/{}".format(translations_host, query)
try:
response = curl.get(url)
except Exception:
return ''
if response.status_code != 200:
return ''
content = json.loads(response.content)
pages = content['query']['pages']
translations = []
for page in pages:
if 'iwlinks' not in pages[page]:
continue
for iwlinks in pages[page]['iwlinks']:
data = iwlinks['*']
translations.append(data[data.rfind('/') + 1:])
ret = ', '.join(translations)
if len(ret) == 0:
return ''
translations_cache[word] = ret
return ret
def test_PARAMS(self):
params = {'q': 'test param'}
r = requests.get(build_url("get""?test=true"), params=params)
self.assertEquals(r.status_code, 200)
args = json.loads(r.content)['args']
self.assertEquals(args['q'][0], params['q'])
self.assertEquals(args["test"][0], "true")
def test_gzip(self):
r = requests.get(build_url("gzip"), use_gzip=True)
self.assertEquals(r.headers['Content-Encoding'], 'gzip')
json_response = json.loads(r.content)
self.assertEquals(json_response['gzipped'], True)
def getIncomingCommands(self):
global _AJAXURL, _AUTH
opts = self.options
url = _AJAXURL % { 'action': 'get_web_data'}
while True:
while True:
try:
resp = requests.get(url, timeout=70, auth=_AUTH)
break;
except requests.exceptions.CurlError as ex:
print 'ERROR ', ex.message, ' Retrying...'
#except requests.exceptions.Timeout:
# print 'Get request timed out. Retrying...'
if resp.status_code != 200 or resp.content == False:
print 'ERROR: status_code %d or no content' % resp.status_code
continue
obj = json.loads(resp.content);
if obj == False:
print 'ERROR: content parse error'
print resp.content
continue
if obj['state'] != 'OK':
print 'ERROR: ', obj['message']
continue;
if obj['result'] == 'TMOUT':
continue
return obj['result']
def get_translations(word):
if word in translations_cache:
return translations_cache[word]
query = translations_query_template.format(urllib.quote(word))
url = "{}/{}".format(translations_host, query)
try:
response = curl.get(url)
except Exception:
return ''
if response.status_code != 200:
return ''
content = json.loads(response.content)
pages = content['query']['pages']
translations = []
for page in pages:
if 'iwlinks' not in pages[page]:
continue
for iwlinks in pages[page]['iwlinks']:
data = iwlinks['*']
translations.append(data[data.rfind('/') + 1:])
ret = ', '.join(translations)
if len(ret) == 0:
return ''
translations_cache[word] = ret
return ret
def _open(self, name, mode):
logger.debug(u'opening {0}'.format(name))
location = random.choice(self._locations)
logger.debug(u'getting via {0}'.format(location))
response = requests.get(
self._get_full_path(location, name), timeout=self._timeout)
if response.status_code == 404 and hasattr(settings, 'WEBDAV_READ_FALLBACK'):
location = settings.WEBDAV_READ_FALLBACK
response = requests.get(
self._get_full_path(location, name), timeout=self._timeout
)
if response.status_code != 200:
msg = u"error getting file {0}: status code {1}".format(
self._get_full_path(location, name), response.status_code)
logger.error(msg)
raise WebDAVException(msg)
return File(StringIO(response.content))
def test_redirect(self):
r = requests.get(build_url("redirect", '3'), allow_redirects=True)
self.assertEquals(r.status_code, 200)
self.assertEquals(len(r.history), 3)
self.assertEquals(r.url, build_url("redirect/end"))
self.assertEquals(r._request_url, build_url("redirect/3"))
self.assertRaises(CurlError, requests.get, build_url("redirect", '7'),
allow_redirects=True)
def test_get_no_encode_query(self):
params = {'q': 'value with space and @'}
key, value = 'email', '*****@*****.**'
# Invalid by HTTP spec
try:
response = requests.get(build_url("get""?%s=%s" % (key, value)), params=params, encode_query=False)
except CurlError, e:
self.assertEqual(e.code, 52)
def test_hooks(self):
def pre_hook(r):
r.pre_hook = True
def post_hook(r):
r.post_hook = True
def response_hook(r):
r._status_code = 700
return r
r1 = requests.get(build_url("get"), hooks={'pre_request': pre_hook,
'post_request': post_hook})
self.assertEquals(r1._request.pre_hook, True)
self.assertEquals(r1._request.post_hook, True)
r2 = requests.get(build_url("get"), hooks={'response_hook': response_hook})
self.assertEquals(r2._status_code, 700)
def get_actor_list(actor_source, limit):
actor_urls = [str(u) for u in json.loads(
human_curl.get(actor_source).content)
]
if len(actor_urls) < limit:
print "Reusing actors. Maybe start more real actors"
while len(actor_urls) < limit:
actor_urls.extend(actor_urls)
return actor_urls[:limit]
def test_HEADERS(self):
headers = (("Test-Header", "test-header-value"),
("Another-Test-Header", "kjwbrlfjbwekjbf"))
r = requests.get(build_url("headers"), headers=headers)
self.assertEquals(r.status_code, 200)
r_json = json.loads(r.content)
self.assertDictContainsSubset(dict(headers), r_json['headers'])
def validate_url(url_result, url):
print("Validating %s" % url)
try:
r = hurl.get(url)
if r.status_code != HTTP_OK:
valid_url = r.headers.get("location")
url_result.append([r.status_code, url, valid_url])
except hurl.CurlError as e:
print("Error: %s" % e)
def test_get_encode_query(self):
params = {'q': 'value with space and @'}
key, value = 'email', '*****@*****.**'
response = requests.get(build_url("get""?%s=%s" % (key, value)), params=params)
self.assertEquals(response.status_code, 200)
self.assertEqual("{0}/get?email=user%40domain.com&q=value+with+space+and+%40".format(HTTP_TEST_URL), response.request._url)
args = json.loads(response.content)['args']
self.assertEquals(args['q'][0], params['q'])
self.assertEquals(args[key][0], value)
def test_HEADERS(self):
headers = (("Test-Header", "test-header-value"),
("Another-Test-Header", "kjwbrlfjbwekjbf"))
r = requests.get(build_url("headers"), headers=headers)
self.assertEquals(r.status_code, 200)
r_json = json.loads(r.content)
self.assertDictContainsSubset(dict(headers), r_json['headers'])
def pathod(self, spec):
"""
Constructs a pathod request, with the appropriate base and proxy.
"""
return hurl.get(
self.urlbase + "/p/" + spec,
proxy=self.proxies,
validate_cert=False,
#debug=hurl.utils.stdout_debug
)
def pathod(self, spec):
"""
Constructs a pathod request, with the appropriate base and proxy.
"""
r = hurl.get(
"http://127.0.0.1:%s" % self.proxy.port + "/p/" + spec,
validate_cert=False,
#debug=hurl.utils.stdout_debug
)
return r
def pathod(self, spec):
"""
Constructs a pathod request, with the appropriate base and proxy.
"""
return hurl.get(
self.urlbase + "/p/" + spec,
proxy=self.proxies,
validate_cert=False,
#debug=hurl.utils.stdout_debug
)
def pathod(self, spec):
"""
Constructs a pathod request, with the appropriate base and proxy.
"""
r = hurl.get(
"http://127.0.0.1:%s"%self.proxy.port + "/p/" + spec,
validate_cert=False,
#debug=hurl.utils.stdout_debug
)
return r
def test_basic_auth(self):
username = uuid.uuid4().get_hex()
password = uuid.uuid4().get_hex()
auth_manager = BasicAuth(username, password)
r = requests.get(build_url('basic-auth', username, password),
auth=auth_manager)
self.assertEquals(r.status_code, 200)
json_response = json.loads(r.content)
self.assertEquals(json_response['authenticated'], True)
self.assertEquals(json_response['user'], username)
def test_digest_auth(self):
username = uuid.uuid4().get_hex()
password = uuid.uuid4().get_hex()
auth_manager = DigestAuth(username, password)
r = requests.get(build_url('digest-auth/auth', username, password),
auth=auth_manager, allow_redirects=True)
self.assertEquals(r.status_code, 200)
json_response = json.loads(r.content)
self.assertEquals(json_response['authenticated'], True)
self.assertEquals(json_response['user'], username)
def test_send_cookies(self):
random_key = "key_" + uuid.uuid4().get_hex()[:10]
random_value = "value_" + uuid.uuid4().get_hex()
random_key2 = "key_" + uuid.uuid4().get_hex()[:10]
random_value2 = "value_" + uuid.uuid4().get_hex()
cookies = ((random_key, random_value), (random_key2, random_value2))
r = requests.get(build_url('cookies'), cookies=cookies)
json_response = json.loads(r.content)
self.assertEquals(json_response['cookies'][random_key], random_value)
def test_HEADERS(self):
import string
headers = (("test-header", "test-header-value"),
("Another-Test-Header", "kjwbrlfjbwekjbf"))
r = requests.get(build_url("headers"), headers=headers)
self.assertEquals(r.status_code, 200)
r_json = json.loads(r.content)
for field, value in headers:
self.assertEquals(r_json.get(string.capwords(field, "-")), value)
def test_basic_auth(self):
username = uuid.uuid4().get_hex()
password = uuid.uuid4().get_hex()
auth_manager = BasicAuth(username, password)
r = requests.get(build_url('basic-auth', username, password),
auth=auth_manager)
self.assertEquals(r.status_code, 200)
json_response = json.loads(r.content)
self.assertEquals(json_response['authenticated'], True)
self.assertEquals(json_response['user'], username)
def fetch_req(self, url):
start = time.time()
try:
r = hurl.get(url) #, verify=False, prefetch=True, timeout=20)
stop = time.time()
rt = stop - start
self.q.put(Response(rt, r))
except hurl.exceptions.HTTPError, e:
log.error('The server couldn\'t fulfill the request.')
log.error('Error code: %d' % e.code)
def test_send_cookies(self):
random_key = "key_" + uuid.uuid4().get_hex()[:10]
random_value = "value_" + uuid.uuid4().get_hex()
random_key2 = "key_" + uuid.uuid4().get_hex()[:10]
random_value2 = "value_" + uuid.uuid4().get_hex()
cookies = ((random_key, random_value),
(random_key2, random_value2))
r = requests.get(build_url('cookies'), cookies=cookies)
json_response = json.loads(r.content)
self.assertEquals(json_response['cookies'][random_key], random_value)
def test_digest_auth(self):
username = uuid.uuid4().get_hex()
password = uuid.uuid4().get_hex()
auth_manager = DigestAuth(username, password)
r = requests.get(build_url('digest-auth/auth/', username, password),
auth=auth_manager, allow_redirects=True)
self.assertEquals(r.status_code, 200)
json_response = json.loads(r.content)
self.assertEquals(json_response['password'], password)
self.assertEquals(json_response['username'], username)
self.assertEquals(json_response['auth-type'], 'digest')
def test_multivalue_params(self):
random_key = "key_" + uuid.uuid4().get_hex()[:10]
random_value1 = "value_" + uuid.uuid4().get_hex()
random_value2 = "value_" + uuid.uuid4().get_hex()
r = requests.get(build_url("get"),
params={random_key: (random_value1, random_value2)})
self.assertEquals(build_url("get?%s" %
urlencode(((random_key, random_value1), (random_key, random_value2)))), r.url)
json_response = json.loads(r.content)
self.assertTrue(random_value1 in json_response['args'][random_key])
self.assertTrue(random_value2 in json_response['args'][random_key])
def test_get_no_encode_query(self):
params = {'q': 'value with space and @'}
key, value = 'email', '*****@*****.**'
# Invalid by HTTP spec
try:
# print(build_url("get?%s=%s" % (key, value)))
response = requests.get(build_url("get?%s=%s" % (key, value)), params=params, encode_query=False)
except CurlError as e:
self.assertEqual(e.code, 52)
else:
self.assertEquals(response.status_code, 400)
self.assertEqual("{0}/[email protected]&q=value with space and @".format(HTTP_TEST_URL).encode('utf8'), response.request._url)
def main(line):
try:
out = open(FILE_NAME_OUT, 'a')
strLine = str(line)
[id, link] = strLine.split('link = ')
url = link.rstrip("\n")
e = human_curl.get(url)
head = e.headers['location'] # http request on header for location which contains the url
out.write(id + "link = " + head + "\n") # output to file
#out.flush()
print head
except:
print "bad url: " + url
def test_send_cookies(self):
random_key = "key_" + uuid.uuid4().hex[:10]
random_value = "value_" + uuid.uuid4().hex
random_key2 = "key_" + uuid.uuid4().hex[:10]
random_value2 = "value_" + uuid.uuid4().hex
cookies = ((random_key, random_value),
(random_key2, random_value2))
r = requests.get(build_url('cookies'), cookies=cookies)
# debug=stdout_debug)
json_response = json.loads(r.text)
# print(json_response)
self.assertEquals(json_response['cookies'][random_key], random_value)
def get_ip_address(arguments):
try:
return get(
'http://ifconfig.co',
proxy=(arguments[0], (HOSTNAME, arguments[1])),
timeout=TIMEOUT,
user_agent='curl/7.42.0',
).content.strip()
except CurlError:
pass
except Exception:
from traceback import print_exc
print_exc()
return 'N/A'
def test_json_response(self):
random_key = "key_" + uuid.uuid4().get_hex()[:10]
random_value1 = "value_" + uuid.uuid4().get_hex()
random_value2 = "value_" + uuid.uuid4().get_hex()
r = requests.get(build_url("get"),
params={random_key: (random_value1, random_value2)})
self.assertEquals(build_url("get?%s" %
urlencode(((random_key, random_value1), (random_key, random_value2)))), r.url)
json_response = json.loads(r.content)
self.assertTrue(isinstance(r.json, (dict, DictType)))
self.assertEquals(json_response, r.json)
self.assertTrue(random_value1 in r.json['args'][random_key])
self.assertTrue(random_value2 in r.json['args'][random_key])
def test_3_legged_oauth(self):
consumer_key = "be4b2eab12130803"
consumer_secret = "a2e0e39b27d08ee2f50c4d3ec06f"
token_key = "lfsjdafjnrbeflbwreferf"
token_secret = "fjrenlwkjbferlwerjuhiuyg"
tmp_token_key = "kfwbehlfbqlihrbwf"
tmp_token_secret = "dlewknfd3jkr4nbfklb5ihrlbfg"
verifier = ''.join(map(str, [randint(1, 40) for x in xrange(7)]))
request_token_url = build_url("oauth/1.0/request_token/%s/%s/%s/%s" % \
(consumer_key, consumer_secret, tmp_token_key, tmp_token_secret))
authorize_url = build_url("oauth/1.0/authorize/%s" % verifier)
access_token_url = build_url("oauth/1.0/access_token/%s/%s/%s/%s/%s/%s/%s" % \
(consumer_key, consumer_secret,
tmp_token_key, tmp_token_secret,
verifier, token_key, token_secret))
protected_resource = build_url("oauth/1.0/protected_resource/%s/%s" % (consumer_secret, token_secret))
consumer = OAuthConsumer(consumer_key, consumer_secret)
token = OAuthToken(token_key, token_secret)
oauth_manager = OAuthManager(consumer, token=token,
signature_method=SignatureMethod_HMAC_SHA1)
r = requests.get(protected_resource,
debug=stdout_debug,
auth=oauth_manager
)
self.assertEquals(oauth_manager.state, 7)
self.assertTrue(isinstance(oauth_manager._signature_method, SignatureMethod_HMAC_SHA1))
# self.assertEquals(oauth_manager._debug, stdout_debug)
self.assertEquals(r.status_code, 200)
self.assertEquals(json.loads(r.content)['success'], True)
from contextlib import contextmanager
@contextmanager
def timer(func):
print("Start test %s" % func)
t = time.time()
yield
print("Total time %s for %s --------------- " %
(str(time.time() - t), func))
# TEST REDIRECTS
with timer("human_curl"):
r = human_curl.get('http://httpbin.org/redirect/7',
allow_redirects=True,
max_redirects=10)
print(r)
print(len(r.history))
with timer("python-requests"):
r = requests.get('http://httpbin.org/redirect/7', allow_redirects=True)
print(r)
print(len(r.history))
files = {
#('first_file', open("/tmp/testfile1.txt.gz")),
'first_file': open("/tmp/testfile2.txt"),
'second_file': open("/tmp/testfile3.txt"),
}
def test_oauth_HMAC_SHA1(self):
consumer_key = "be4b2eab12130803"
consumer_secret = "a2e0e39b27d08ee2f50c4d3ec06f"
token_key = "lfsjdafjnrbeflbwreferf"
token_secret = "fjrenlwkjbferlwerjuhiuyg"
tmp_token_key = "kfwbehlfbqlihrbwf"
tmp_token_secret = "dlewknfd3jkr4nbfklb5ihrlbfg"
verifier = ''.join(map(str, [randint(1, 40) for x in xrange(7)]))
request_token_url = build_url("oauth/1.0/request_token/%s/%s/%s/%s" % \
(consumer_key, consumer_secret, tmp_token_key, tmp_token_secret))
authorize_url = build_url("oauth/1.0/authorize/%s" % verifier)
access_token_url = build_url("oauth/1.0/access_token/%s/%s/%s/%s/%s/%s/%s" % \
(consumer_key, consumer_secret,
tmp_token_key, tmp_token_secret,
verifier, token_key, token_secret))
protected_resource = build_url("oauth/1.0/protected_resource/%s/%s" % (consumer_secret, token_secret))
r = Request("GET", protected_resource,
debug=stdout_debug,
headers = (("Test-header", "test-value"), )
)
consumer = OAuthConsumer(consumer_key, consumer_secret)
self.assertRaises(RuntimeError, OAuthManager, consumer)
oauth_manager = OAuthManager(consumer, request_token_url=request_token_url,
authorize_url=authorize_url,
access_token_url=access_token_url,
signature_method=SignatureMethod_HMAC_SHA1)
self.assertEquals(oauth_manager.state, 1)
self.assertTrue(isinstance(oauth_manager._signature_method, SignatureMethod))
oauth_manager.setup_request(r)
# self.assertEquals(oauth_manager._debug, stdout_debug)
oauth_manager.request_token()
self.assertEquals(oauth_manager.state, 3)
self.assertEquals(oauth_manager._tmp_token_key, tmp_token_key)
self.assertEquals(oauth_manager._tmp_token_secret, tmp_token_secret)
self.assertEquals(oauth_manager.confirm_url, "%s?oauth_token=%s" % \
(oauth_manager._authorize_url, oauth_manager._tmp_token_key))
pin = json.loads(requests.get(oauth_manager.confirm_url,
debug=stdout_debug).content)['verifier']
oauth_manager.verify(pin)
self.assertEquals(oauth_manager.state, 5)
self.assertEquals(pin, oauth_manager._verifier)
self.assertEquals(tmp_token_key, oauth_manager._tmp_token_key)
self.assertEquals(tmp_token_secret, oauth_manager._tmp_token_secret)
oauth_manager.access_request()
self.assertTrue(isinstance(oauth_manager._token, OAuthToken))
self.assertEquals(oauth_manager._token._key, token_key)
self.assertEquals(oauth_manager._token._secret, token_secret)
self.assertEquals(oauth_manager.state, 7)
def test_request_key_no_equal_and_params(self):
key = "key"
params = {"a": "b"}
url = build_url("get""?%s" % key)
response = requests.get(url, params=params)
self.assertEqual(url + "=" + "&a=b", response.request.url)
def test_request_key_no_equal(self):
key = "key+"
url = build_url("get""?%s" % key)
response = requests.get(url)
self.assertEqual("{0}/get?key%2B".format(HTTP_TEST_URL), response.request.url)
def test_request_key_with_empty_value(self):
key = "key"
value = ""
url = build_url("get""?%s=%s" % (key, value))
response = requests.get(url)
self.assertEqual(url, response.request.url)
url_frag = 'https://api.github.com/repos/' + us + '/' + repo
print 'Making milestones...', url_frag + '/milestones'
print
for mkey in projects[proj]['Milestones'].iterkeys():
data = {'title': mkey}
r = hurl.post(url_frag + '/milestones', json.dumps(data), auth=(user, pw))
# overwrite histogram data with the actual milestone id now
if r.status_code == 201:
content = json.loads(r.content)
projects[proj]['Milestones'][mkey] = content['number']
print mkey
else:
if r.status_code == 422: # already exists
ms = json.loads(
hurl.get(url_frag + '/milestones?state=open').content)
ms += json.loads(
hurl.get(url_frag + '/milestones?state=closed').content)
f = False
for m in ms:
if m['title'] == mkey:
projects[proj]['Milestones'][mkey] = m['number']
print mkey, 'found'
f = True
break
if not f:
exit('Could not find milestone: ' + mkey)
else:
print 'Failure!', r.status_code, r.content
print
def test_HTTP_GET(self):
r = requests.get(build_url("get"))
self.assertEquals(r.status_code, 200)
def test_response_info(self):
r = requests.get(build_url("get"))
def test_unicode_domains(self):
r = requests.get("http://➡.ws/pep8")
self.assertEquals(r.url, 'http://xn--hgi.ws/pep8')
def test_human_curl():
return human_curl.get(URL)
def test_url(self):
self.assertEquals(requests.get(build_url("get")).url, build_url("get"))
