def main(reactor):
print 'List of tuples'
resp = yield treq.get('http://httpbin.org/get',
params=[('foo', 'bar'), ('baz', 'bax')])
content = yield treq.content(resp)
print content
print 'Single value dictionary'
resp = yield treq.get('http://httpbin.org/get',
params={'foo': 'bar', 'baz': 'bax'})
content = yield treq.content(resp)
print content
print 'Multi value dictionary'
resp = yield treq.get('http://httpbin.org/get',
params={'foo': ['bar', 'baz', 'bax']})
content = yield treq.content(resp)
print content
print 'Mixed value dictionary'
resp = yield treq.get('http://httpbin.org/get',
params={'foo': ['bar', 'baz'], 'bax': 'quux'})
content = yield treq.content(resp)
print content
print 'Preserved query parameters'
resp = yield treq.get('http://httpbin.org/get?foo=bar',
params={'baz': 'bax'})
content = yield treq.content(resp)
print content
def login(self, email, password, accountType = ACCOUNT_TYPE_HOSTED_OR_GOOGLE):
params = {"Email": email, "Passwd": password, "service": self.SERVICE,
"accountType": accountType, "has_permission": "1",
"source": "android", "androidId": self.androidId,
"app": "com.android.vending", "sdk_version": "16" }
resp = yield treq.post(self.URL_LOGIN, params)
if resp.code == http.OK:
data = yield treq.content(resp)
data = data.split()
params = {}
for d in data:
k, v = d.split("=")
params[k.strip()] = v.strip()
if "Auth" in params:
self.setAuthSubToken(params["Auth"])
self.loggedIn = True
else:
raise LoginError("Auth token not found.")
else:
if resp.code == http.FORBIDDEN:
data = yield treq.content(resp)
params = {}
for d in data.split('\n'):
d = d.strip()
if d:
k, v = d.split("=", 1)
params[k.strip()] = v.strip()
if "error" in params:
raise LoginError(params["error"])
else:
raise LoginError("Login failed.")
else:
data = yield treq.content(resp)
raise LoginError("Login failed: error %d <%s>" % (resp.code, data.rstrip(),))
def test_customParameterValidation(self):
# type: () -> None
"""
When a custom parameter fails to validate by raising ValueError - for
example, a non-number passed to a numeric Field, the request fails with
a 400 and the default validation failure handler displays a form which
explains the error.
"""
router, calls = simpleFormRouter()
stub = StubTreq(router.resource())
response = self.successResultOf(
stub.get(b"https://localhost/getme?"
b"name=hello,%20big+world&value=0&custom=not+a+number"))
responseForm = self.successResultOf(content(response))
self.assertEqual(response.code, 400)
self.assertEqual(calls, [])
responseForm = self.successResultOf(content(response))
responseDom = ElementTree.fromstring(responseForm)
errors = responseDom.findall(
".//*[@class='klein-form-validation-error']")
self.assertEqual(len(errors), 1)
errorText = cast(str, errors[0].text)
self.assertIsNot(errorText, None)
self.assertEqual(
errorText,
"invalid literal for int() with base 10: 'not a number'",
)
def test_missingOptionalParameterJSON(self):
# type: () -> None
"""
If a required Field is missing from the JSON body, its default value is
used.
"""
mem = MemorySessionStore()
session = self.successResultOf(
mem.newSession(True, SessionMechanism.Header))
to = TestObject(mem)
stub = StubTreq(to.router.resource())
response = self.successResultOf(
stub.post(
"https://localhost/notrequired",
json=dict(name="one"),
headers={b"X-Test-Session": session.identifier},
))
response2 = self.successResultOf(
stub.post(
"https://localhost/notrequired",
json=dict(name="two", value=2),
headers={b"X-Test-Session": session.identifier},
))
self.assertEqual(response.code, 200)
self.assertEqual(response2.code, 200)
self.assertEqual(self.successResultOf(content(response)), b"okay")
self.assertEqual(self.successResultOf(content(response2)), b"okay")
self.assertEqual(to.calls, [("one", 7.0), ("two", 2.0)])
def _get_using_ipify_ipstack():
"""
The last resort is to ask someone for out external IP address, and then feed this to IPStack
to get more details.
"""
response = yield treq.get(IPIFY_API)
content = yield treq.content(response)
ip = content.decode().strip()
url = MAXMIND_API.format(ip)
try:
response = yield treq.get(url, timeout=5)
except Exception:
return None
content = yield treq.content(response)
raw_info = data_unpickle(content, "json")
return {
"source": "ipify_ipstack",
"ip": raw_info.get("query"),
"country_code": raw_info.get("country_code"),
"country_name": raw_info.get("country_name"),
"region_code": raw_info.get("region_code"),
"region_name": raw_info.get("region_name"),
"city": raw_info.get("city"),
"zip_code": raw_info.get("zip"),
"time_zone": raw_info.get("time_zone", {}).get("id"),
"latitude": float(raw_info.get("latitude")),
"longitude": float(raw_info.get("longitude")),
"isp": raw_info.get("connection", {}).get("isp"),
}
def upload(self, local_path):
log.debug("Uploading %s...", local_path)
with open(local_path, 'rb') as f:
resp = yield treq.put('{}uri'.format(self.nodeurl), f)
if resp.code == 200:
content = yield treq.content(resp)
log.debug("Successfully uploaded %s", local_path)
return content.decode('utf-8')
content = yield treq.content(resp)
raise TahoeWebError(content.decode('utf-8'))
def upload(self, local_path):
log.debug("Uploading %s...", local_path)
yield self.await_ready()
with open(local_path, "rb") as f:
resp = yield treq.put("{}uri".format(self.nodeurl), f)
if resp.code == 200:
content = yield treq.content(resp)
log.debug("Successfully uploaded %s", local_path)
return content.decode("utf-8")
content = yield treq.content(resp)
raise TahoeWebError(content.decode("utf-8"))
def test_content_multiple_waiters(self):
d1 = content(self.response)
d2 = content(self.response)
self.protocol.dataReceived(b'foo')
self.protocol.connectionLost(Failure(ResponseDone()))
self.assertEqual(self.successResultOf(d1), b'foo')
self.assertEqual(self.successResultOf(d2), b'foo')
self.assertNotIdentical(d1, d2)
def test_content_multiple_waiters(self):
d1 = content(self.response)
d2 = content(self.response)
self.protocol.dataReceived('foo')
self.protocol.connectionLost(Failure(ResponseDone()))
self.assertEqual(self.successResultOf(d1), 'foo')
self.assertEqual(self.successResultOf(d2), 'foo')
self.assertNotIdentical(d1, d2)
def handleArchivePage(response):
global CURRENTIP
log.msg( "Archive page code is %s" % response.code)
if response.code ==403:
log.msg("Oh oh - looks like we're in the doghouse")
CURRENTIP=next(IPADDRS)
log.msg("Switching IP to %s" % str(CURRENTIP))
# raise DoghouseException("Couldn't get archive - code was %i" % response.code)
# treq seems to "block" if you don't fetch the content
return treq.content(response)
# return defer.succeed(False)
else:
d=treq.content(response).addCallback(parseArchiveResults)
return d
def test_client_init(self):
# FIXME: this test passes because stuff in the server is hard coded.
# revisit this when we're actually parsing stuff.
syncml = self.syncml.build_request()
response = yield self.request(syncml.build().toXml())
body = yield content(response)
self.assertTrue(str(constants.AUTHENTICATION_ACCEPTED) in body)
def link(self, dircap, childname, childcap):
dircap_hash = trunchash(dircap)
childcap_hash = trunchash(childcap)
log.debug(
'Linking "%s" (%s) into %s...',
childname,
childcap_hash,
dircap_hash,
)
yield self.await_ready()
yield self.lock.acquire()
try:
resp = yield treq.post("{}uri/{}/?t=uri&name={}&uri={}".format(
self.nodeurl, dircap, childname, childcap))
finally:
yield self.lock.release()
if resp.code != 200:
content = yield treq.content(resp)
raise TahoeWebError(content.decode("utf-8"))
log.debug(
'Done linking "%s" (%s) into %s',
childname,
childcap_hash,
dircap_hash,
)
def http_status_filter(response, allowed, ex, msg=None, include_resp_text=True):
"""
Checks the response status and determines if it is in one of the
allowed ranges. If not, it raises `ex()`.
`ex` is a callable that results in an Exception to be raised,
(typically an exception class).
`allowed` is a sequence of (start, end) valid status ranges.
"""
code = response.code
in_range = False
for start_range, end_range in allowed:
if code >= start_range and code <= end_range:
in_range = True
break
if not in_range:
def raise_error(body, ex):
ex_msg = []
if msg is not None:
ex_msg.append(msg)
if include_resp_text:
ex_msg.append(body)
text = '\n'.join(ex_msg)
if text != "":
raise ex(text)
else:
raise ex()
# Need to still deliver the response body or Twisted make
# hang.
d = treq.content(response)
d.addCallback(raise_error, ex)
return d
return response
def message_scan_url(self, url_info):
# I'm still not sure whether I like inlineCallbacks. It made
# this easier to test, though, because of implicit maybeDeferred.
# TODO: limit size of retrieved content. it could be huge!
try:
queue_entry = util.queue_entry_parse(url_info)
print "making request..."
response = yield treq.get(queue_entry.url)
content = yield treq.content(response)
print "got content"
base_url, urls = self.scan_page(queue_entry.url, content)
urls = self.normalize_urls(base_url, urls)
# TODO: would be cool to put some jitter in here. probably not needed
# though, there will be natural jitter from the requests maybe possibly
# hopefully... there will probably still be situations that it'd be
# wanted though
except:
import traceback
traceback.print_exc()
print "offending url:", queue_entry.url
self.command("url_completed", queue_entry.job_id)
return
self.command("url_completed", queue_entry.job_id)
for url in urls:
# blah, QueueEntry feels like java
qe = util.QueueEntry(queue_entry.job_id, queue_entry.level + 1, url)
self.command("found", util.queue_entry_format(qe))
def matches_response(code_matcher=Always(),
headers_matcher=Always(),
body_matcher=Always()):
"""
Match a Treq response object with certain code and body.
:param Matcher code_matcher: A matcher to apply to the response code.
:param Matcher headers_matcher: A matcher to apply to the response headers
(a ``twisted.web.http_headers.Headers`` instance).
:param Matcher body_matcher: A matcher to apply to the response body.
:return: A matcher.
"""
return MatchesAll(
MatchesStructure(
code=code_matcher,
headers=headers_matcher,
),
AfterPreprocessing(
lambda response: content(response),
succeeded(body_matcher),
),
)
def createSwiftService(self, rackspace_flavor=True):
"""
Set up to create the requests
"""
self.swift_mock = SwiftMock(rackspace_flavor)
self.core = MimicCore(Clock(), [self.swift_mock])
self.root = MimicRoot(self.core).app.resource()
self.response = request(
self,
self.root,
b"POST",
b"/identity/v2.0/tokens",
dumps({
"auth": {
"passwordCredentials": {
"username": "******",
"password": "******",
},
# TODO: should this really be 'tenantId'?
"tenantName": "fun_tenant",
}
}).encode("utf-8"))
self.auth_response = self.successResultOf(self.response)
text_body = self.successResultOf(treq.content(
self.auth_response)).decode("utf-8")
self.json_body = loads(text_body)
def print_response(r):
print(r.code, r.phrase)
print(r.headers)
d = treq.content(r)
d.addCallback(print)
return d
def got_result(result):
if result.code == success_code:
return json_content(result)
else:
d = content(result)
d.addCallback(error, result.code)
return d
def matches_response(code_matcher=Always(),
headers_matcher=Always(),
body_matcher=Always()):
"""
Match a Treq response object with certain code and body.
:param Matcher code_matcher: A matcher to apply to the response code.
:param Matcher headers_matcher: A matcher to apply to the response headers
(a ``twisted.web.http_headers.Headers`` instance).
:param Matcher body_matcher: A matcher to apply to the response body.
:return: A matcher.
"""
matchers = [
MatchesStructure(
code=code_matcher,
headers=headers_matcher,
),
]
# see comment in test_web.MagicFolderTests.test_method_not_allowed
# which is one user that wants nothing to try and read the content
# in some cases..
if body_matcher is not None:
matchers.append(
AfterPreprocessing(
lambda response: content(response),
succeeded(body_matcher),
))
return MatchesAll(*matchers)
def test_delete_server_fails_specified_number_of_times(self):
"""
Test to verify :func: `delete_server` does not delete the server,
and returns the given response code, the number of times specified
in the metadata
"""
deletefail = {"times": 1, "code": 500}
metadata = {"delete_server_failure": json.dumps(deletefail)}
# create server and verify it was successful
create_server_response = self.create_server(metadata=metadata)
self.assertEquals(create_server_response.code, 202)
create_server_response_body = self.successResultOf(
treq.json_content(create_server_response))
# delete server and verify the response
delete_server = request(self, self.root, "DELETE", self.uri + '/servers/'
+ create_server_response_body["server"]["id"])
delete_server_response = self.successResultOf(delete_server)
self.assertEqual(delete_server_response.code, 500)
# get server and verify the server was not deleted
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
self.assertEquals(get_server_response.code, 200)
# delete server again and verify the response
delete_server = request(self, self.root, "DELETE", self.uri + '/servers/'
+ create_server_response_body["server"]["id"])
delete_server_response = self.successResultOf(delete_server)
self.assertEqual(delete_server_response.code, 204)
self.assertEqual(self.successResultOf(treq.content(delete_server_response)),
b"")
# get server and verify the server was deleted this time
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
self.assertEquals(get_server_response.code, 404)
def test_delete_server_fails_specified_number_of_times(self):
"""
Test to verify :func: `delete_server` does not delete the server,
and returns the given response code, the number of times specified
in the metadata
"""
deletefail = {"times": 1, "code": 500}
metadata = {"delete_server_failure": json.dumps(deletefail)}
# create server and verify it was successful
create_server_response = self.create_server(metadata=metadata)
self.assertEquals(create_server_response.code, 202)
create_server_response_body = self.successResultOf(
treq.json_content(create_server_response))
# delete server and verify the response
delete_server = request(self, self.root, "DELETE", self.uri + '/servers/'
+ create_server_response_body["server"]["id"])
delete_server_response = self.successResultOf(delete_server)
self.assertEqual(delete_server_response.code, 500)
# get server and verify the server was not deleted
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
self.assertEquals(get_server_response.code, 200)
# delete server again and verify the response
delete_server = request(self, self.root, "DELETE", self.uri + '/servers/'
+ create_server_response_body["server"]["id"])
delete_server_response = self.successResultOf(delete_server)
self.assertEqual(delete_server_response.code, 204)
self.assertEqual(self.successResultOf(treq.content(delete_server_response)),
b"")
# get server and verify the server was deleted this time
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
self.assertEquals(get_server_response.code, 404)
def tts(input_xml, affect='surprise', ptype='disagree'):
mary_host = "172.17.0.1"
mary_port = "59125"
maryxml = f'''<?xml version="1.0" encoding="UTF-8" ?>
<maryxml version="0.4"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns=" http://mary.dfki.de/2002/MaryXML"
xml:lang="en-US">
<p>
{input_xml}
</p>
</maryxml>'''
query_hash = {
"INPUT_TEXT": maryxml,
"INPUT_TYPE": "RAWMARYXML",
"LOCALE": "en_US",
"VOICE": "cmu-slt-hsmm",
"OUTPUT_TYPE": "AUDIO",
"AUDIO": "WAVE",
}
query = urlencode(query_hash).replace('+', '%20')
resp = yield treq.post("http://%s:%s/process?" % (mary_host, mary_port) +
query)
wav = yield treq.content(resp)
which = '%.1f' % time.time()
recentFiles[which] = wav
url = f'http://10.2.0.1:9010/data/{which}'
print('save', len(wav), 'bytes at', url)
return url.encode('ascii')
def got_result(result):
if result.code == success_code:
return json_content(result)
else:
d = content(result)
d.addCallback(error, result.code)
return d
def make_request():
if ssl:
response = yield treq.post(url, agent=self.agent, **kwargs)
else:
response = yield treq.post(url, **kwargs)
try:
content = yield treq.content(response)
except:
log.msg("Error while reading body in HTTP response",
level=logger.WARN)
self.stats_client.incr('http.requests.errors.read_body_error')
err = RuntimeError(
"Error while reading body in HTTP response (Response code: %d)."
% response.code)
err.status_code = response.code
raise err
else:
if response.code == 200:
defer.returnValue(content)
else:
log.msg("Error: Bad status code in HTTP response",
http_response_code=response.code,
http_content=content,
level=logger.WARN)
self.stats_client.incr(
'http.requests.errors.bad_status_code.%d' %
response.code)
err = RuntimeError(
'Error: Bad status code in HTTP response: %d' %
response.code)
err.status_code = response.code
raise err
def check_upstream_response(response, request):
content = yield treq.content(response)
if response.code != 200:
request.setResponseCode(502)
defer.returnValue("Upstream response code: {:d}\n{!s}".format(response.code, content))
else:
defer.returnValue(content)
def callRemote(self, method, *args):
scheme = "https://" if self.secure else "http://"
netloc = self.host
if self.port:
netloc += ":%s" % self.port
if self.user:
auth = self.user
if self.password:
auth += ":%s" % self.password
netloc = "%s@%s" % (auth, netloc)
url = scheme + netloc + self.path
data = self.payloadTemplate % (
method,
xmlrpclib.dumps(args, allow_none=self.allowNone),
)
headers = {"User-Agent": "Twisted/XMLRPC", "Content-Type": "text/xml"}
response = yield treq.post(
url,
data,
headers=headers,
timeout=self.connectTimeout,
reactor=self._reactor,
)
body = yield treq.content(response)
parsed = xmlrpclib.loads(body, use_datetime=self.useDateTime)[0][0]
returnValue(parsed)
def test_numberConstraints(self):
# type: () -> None
"""
Number parameters have minimum and maximum validations and the object
will not be called when the values exceed them.
"""
mem = MemorySessionStore()
session = self.successResultOf(
mem.newSession(True, SessionMechanism.Header))
to = TestObject(mem)
stub = StubTreq(to.router.resource())
tooLow = self.successResultOf(
stub.post('https://localhost/constrained',
data=dict(goldilocks='1'),
headers={b'X-Test-Session': session.identifier}))
tooHigh = self.successResultOf(
stub.post('https://localhost/constrained',
data=dict(goldilocks='20'),
headers={b'X-Test-Session': session.identifier}))
justRight = self.successResultOf(
stub.post('https://localhost/constrained',
data=dict(goldilocks='7'),
headers={b'X-Test-Session': session.identifier}))
self.assertEqual(tooHigh.code, 400)
self.assertEqual(tooLow.code, 400)
self.assertEqual(justRight.code, 200)
self.assertEqual(self.successResultOf(content(justRight)), b'got it')
self.assertEqual(to.calls, [(u'constrained', 7)])
def message_scan_url(self, url_info):
# I'm still not sure whether I like inlineCallbacks. It made
# this easier to test, though, because of implicit maybeDeferred.
# TODO: limit size of retrieved content. it could be huge!
try:
queue_entry = util.queue_entry_parse(url_info)
print "making request..."
response = yield treq.get(queue_entry.url)
content = yield treq.content(response)
print "got content"
base_url, urls = self.scan_page(queue_entry.url, content)
urls = self.normalize_urls(base_url, urls)
# TODO: would be cool to put some jitter in here. probably not needed
# though, there will be natural jitter from the requests maybe possibly
# hopefully... there will probably still be situations that it'd be
# wanted though
except:
import traceback
traceback.print_exc()
print "offending url:", queue_entry.url
self.command("url_completed", queue_entry.job_id)
return
self.command("url_completed", queue_entry.job_id)
for url in urls:
# blah, QueueEntry feels like java
qe = util.QueueEntry(queue_entry.job_id, queue_entry.level + 1,
url)
self.command("found", util.queue_entry_format(qe))
def start(self):
host = "http://%(host)s"%(self.args)
res = "/nexus/service/local/artifact/maven/content"
artifact = self.args["artifact"]
fileName = self.args["file"]
fileName = os.path.join(self.builder.basedir,fileName)
g,a,v,e = artifact.split(":")
params = {
'r': self.args["repo"],
'g': g,
'a': a,
'v': v,
'e': e,
'p': e}
print "posting",host+res, params, fileName,self.args["cred"]
res = yield treq.post(
host+res,
data = params,
files = {'file':open(fileName,"rb")},
auth=self.args["cred"])
print "res",res
output = yield treq.content(res)
print "info", res.code, output
self.sendStatus({'info':(res.code,output)})
self.sendStatus({'rc': 0 if res.code == 201 else 1})
def download_file(self, url: str, destination_filename: str):
"""
Downloads a file from the given url and saves it to the requested destination.
Returns a deferred!
:param url:
:param destination_filename:
:return:
"""
try:
treq_response = yield treq.request("GET", url)
except ConnectionRefusedError as e:
raise YomboWarning(f"Connection was refused to '{url}': {e}")
except ConnectError as e:
raise YomboWarning(f"Error connecting to '{url}': {e}")
except Exception as e:
logger.info("Requests download_file error: {error}", error=e)
logger.error(
"---------------==(Traceback)==--------------------------")
logger.error("{url}", url=url)
logger.error("{trace}", trace=traceback.format_exc())
logger.error(
"--------------------------------------------------------")
logger.warn(
"An exception of type {etype} occurred in yombo.lib.yomboapi:import_component. Message: {msg}",
etype=type(e),
msg=e)
logger.error(
"--------------------------------------------------------")
raise e
raw_content = yield treq.content(treq_response)
yield self._Files.save(destination_filename, raw_content)
def get_grid_status(self):
if not self.nodeurl:
return
try:
resp = yield treq.get(self.nodeurl + '?t=json') # not yet released
except ConnectError:
return
if resp.code == 200:
content = yield treq.content(resp)
content = content.decode('utf-8')
try:
content = json.loads(content)
except json.decoder.JSONDecodeError:
# See: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2476
connected, known, space = self._parse_welcome_page(content)
returnValue((connected, known, space))
servers_connected = 0
servers_known = 0
available_space = 0
if 'servers' in content:
servers = content['servers']
servers_known = len(servers)
for server in servers:
if server['connection_status'].startswith('Connected'):
servers_connected += 1
if server['available_space']:
available_space += server['available_space']
returnValue((servers_connected, servers_known, available_space))
def test_renderingFormGlue(self):
# type: () -> None
"""
When a form renderer renders just the glue, none of the rest of the
form is included.
"""
mem = MemorySessionStore()
session = self.successResultOf(
mem.newSession(True, SessionMechanism.Header))
stub = StubTreq(TestObject(mem).router.resource())
response = self.successResultOf(
stub.get(
"https://localhost/render-custom",
headers={b"X-Test-Session": session.identifier},
))
self.assertEqual(response.code, 200)
self.assertIn(
response.headers.getRawHeaders(b"content-type")[0], b"text/html")
responseDom = ElementTree.fromstring(
self.successResultOf(content(response)))
submitButton = responseDom.findall(".//*[@type='submit']")
self.assertEqual(len(submitButton), 0)
protectionField = responseDom.findall(
".//*[@name='__csrf_protection__']")
self.assertEqual(protectionField[0].attrib["value"],
session.identifier)
def test_missingRequiredParameter(self):
# type: () -> None
"""
If required fields are missing, a default error form is presented and
the form's handler is not called.
"""
mem = MemorySessionStore()
session = self.successResultOf(
mem.newSession(True, SessionMechanism.Header))
to = TestObject(mem)
stub = StubTreq(to.router.resource())
response = self.successResultOf(
stub.post(
"https://localhost/handle",
data=dict(),
headers={b"X-Test-Session": session.identifier},
))
self.assertEqual(response.code, 400)
self.assertIn(
b"a value was required but none was supplied",
self.successResultOf(content(response)),
)
self.assertEqual(to.calls, [])
def test_customValidationHandling(self):
# type: () -> None
"""
L{Form.onValidationFailureFor} handles form validation failures by
handing its thing a renderable form.
"""
mem = MemorySessionStore()
session = self.successResultOf(
mem.newSession(True, SessionMechanism.Header))
testobj = TestObject(mem)
stub = StubTreq(testobj.router.resource())
response = self.successResultOf(
stub.post(
"https://localhost/handle-validation",
headers={b"X-Test-Session": session.identifier},
json={"value": 300},
))
self.assertEqual(response.code, 200)
self.assertIn(
response.headers.getRawHeaders(b"content-type")[0], b"text/html")
responseText = self.successResultOf(content(response))
self.assertEqual(responseText, b"~special~")
self.assertEqual(
[(k.pythonArgumentName, v)
for k, v in testobj.calls[-1][1].prevalidationValues.items()],
[("value", 300)],
)
def make_request():
if ssl:
response = yield treq.post(url, agent=self.agent, **kwargs)
else:
response = yield treq.post(url, **kwargs)
try:
content = yield treq.content(response)
except:
log.msg("Error while reading body in HTTP response",
level=logger.WARN)
self.stats_client.incr('http.requests.errors.read_body_error')
err = RuntimeError("Error while reading body in HTTP response (Response code: %d)." % response.code)
err.status_code = response.code
raise err
else:
if response.code == 200:
defer.returnValue(content)
else:
log.msg("Error: Bad status code in HTTP response",
http_response_code=response.code,
http_content=content,
level=logger.WARN)
self.stats_client.incr('http.requests.errors.bad_status_code.%d' % response.code)
err = RuntimeError('Error: Bad status code in HTTP response: %d' % response.code)
err.status_code = response.code
raise err
def test_client_init(self):
# FIXME: this test passes because stuff in the server is hard coded.
# revisit this when we're actually parsing stuff.
syncml = self.syncml.build_request()
response = yield self.request(syncml.build().toXml())
body = yield content(response)
self.assertTrue(str(constants.AUTHENTICATION_ACCEPTED) in body)
def test_put_object(self):
"""
PUTting an object into a container causes the container to list that
object.
"""
self.createSwiftService()
# create a container
uri = (self.json_body['access']['serviceCatalog'][0]['endpoints'][0]
['publicURL'] + '/testcontainer')
create_container = request(self, self.root, "PUT", uri)
self.successResultOf(create_container)
BODY = b'some bytes'
object_uri = uri + "/" + "testobject"
object_response = request(self,
self.root,
"PUT",
object_uri,
headers={"content-type": ["text/plain"]},
body=BODY)
self.assertEqual(self.successResultOf(object_response).code, 201)
container_response = self.successResultOf(
request(self, self.root, "GET", uri))
self.assertEqual(container_response.code, 200)
container_contents = self.successResultOf(
treq.json_content(container_response))
self.assertEqual(len(container_contents), 1)
self.assertEqual(container_contents[0]['name'], "testobject")
self.assertEqual(container_contents[0]['content_type'], "text/plain")
self.assertEqual(container_contents[0]['bytes'], len(BODY))
object_response = self.successResultOf(
request(self, self.root, "GET", object_uri))
self.assertEqual(object_response.code, 200)
object_body = self.successResultOf(treq.content(object_response))
self.assertEquals(object_body, BODY)
def test_cookieWithToken(self):
# type: () -> None
"""
A cookie-authenticated, CRSF-protected form will call the form as
expected.
"""
mem = MemorySessionStore()
session = self.successResultOf(
mem.newSession(True, SessionMechanism.Cookie))
to = TestObject(mem)
stub = StubTreq(to.router.resource())
response = self.successResultOf(
stub.post(
"https://localhost/handle",
data=dict(
name="hello",
value="1234",
ignoreme="extraneous",
__csrf_protection__=session.identifier,
),
cookies={
"Klein-Secure-Session": nativeString(session.identifier)
},
))
self.assertEqual(to.calls, [("hello", 1234)])
self.assertEqual(response.code, 200)
self.assertIn(b"yay", self.successResultOf(content(response)))
def _delete_ticket(self, _id, _rev):
"""
Delete a ticket from CouchDB.
"""
url = '''%(scheme)s%(host)s:%(port)s/%(db)s/%(docid)s''' % {
'scheme': self._scheme,
'host': self._couch_host,
'port': self._couch_port,
'db': self._couch_db,
'docid': _id
}
url = url.encode('utf-8')
params = {'rev': _rev}
self.debug('[DEBUG][CouchDB] _delete_ticket(), url: %s' % url)
self.debug('[DEBUG][CouchDB] _delete_ticket(), params: %s' %
str(params))
httpClient = self.httpClientFactory(self.reactor)
response = yield httpClient.delete(
url,
params=params,
auth=(self._couch_user, self._couch_passwd),
headers=Headers({'Accept': ['application/json']}))
response = yield http_status_filter(response, [(200, 200)],
CouchDBError)
resp_text = yield treq.content(response)
defer.returnValue(None)
def get_external_ip_address_v4():
"""
Get the IP address of this machine as seen from the outside world. THis
function is primarily used during various internal testing of the Yombo
Gateway. This information is reported to the Yombo Service, however, the
Yombo Service already knows you're IP address during the process of
downloading configuration files.
Yombo servers will only use this information if server "getpeer.ip()" function
results in a private IP address. See: http://en.wikipedia.org/wiki/Private_network
This assists in Yombo performing various tests internally, but still providing
an ability to do further tests.
Gives Yombo servers a hint of your external ip address from your view. This
should be the same as what Yombo servers see when you connect.
This is called only once during the startup phase. Calling this function too
often can result in the gateway being blocked by whatismyip.org
.. warning::
This is 100% blocking code. A replacement will be coming soon.
:return: An ip address
:rtype: string
"""
import treq
# response = yield treq.get("https://yombo.net/tools/clientip.php") # fails check on older systems due to wildcard
response = yield treq.get("https://api.ipify.org")
content = yield treq.content(response)
return content.decode().strip()
def test_rendering(self):
# type: () -> None
"""
When a route requires form fields, it renders a form with those fields.
"""
mem = MemorySessionStore()
session = self.successResultOf(
mem.newSession(True, SessionMechanism.Header))
stub = StubTreq(TestObject(mem).router.resource())
response = self.successResultOf(
stub.get(
"https://localhost/render",
headers={b"X-Test-Session": session.identifier},
))
self.assertEqual(response.code, 200)
self.assertIn(
response.headers.getRawHeaders(b"content-type")[0], b"text/html")
responseDom = ElementTree.fromstring(
self.successResultOf(content(response)))
submitButton = responseDom.findall(".//*[@type='submit']")
self.assertEqual(len(submitButton), 1)
self.assertEqual(submitButton[0].attrib["name"],
"__klein_auto_submit__")
def test_renderingEmptyForm(self):
# type: () -> None
"""
When a form renderer specifies a submit button, no automatic submit
button is rendered.
"""
mem = MemorySessionStore()
session = self.successResultOf(
mem.newSession(True, SessionMechanism.Header))
stub = StubTreq(TestObject(mem).router.resource())
response = self.successResultOf(
stub.get(
"https://localhost/render-empty",
headers={b"X-Test-Session": session.identifier},
))
self.assertEqual(response.code, 200)
self.assertIn(
response.headers.getRawHeaders(b"content-type")[0], b"text/html")
responseDom = ElementTree.fromstring(
self.successResultOf(content(response)))
submitButton = responseDom.findall(".//*[@type='submit']")
self.assertEqual(len(submitButton), 1)
self.assertEqual(submitButton[0].attrib["name"],
"__klein_auto_submit__")
protectionField = responseDom.findall(
".//*[@name='__csrf_protection__']")
self.assertEqual(protectionField[0].attrib["value"],
session.identifier)
def test_invalid_content_type(self):
response = yield self.request_with_fixture('client_sync_init.xml', {
'Content-Type': ['foo'],
})
body = yield content(response)
self.assertEqual(response.code, http.NOT_ACCEPTABLE)
self.assertContentType(response, 'text/plain')
self.assertEqual(body, 'Unsupported content-type.')
def test_delete_queue(self):
"""
Test to verify :func:`del_queue` on ``DELETE /v2.0/<tenant_id>/servers/<queue_name>``
"""
delete_queue = request(self, self.root, b"DELETE", self.uri + '/queues/' + self.queue_name)
delete_queue_response = self.successResultOf(delete_queue)
self.assertEqual(delete_queue_response.code, 204)
self.assertEqual(self.successResultOf(treq.content(delete_queue_response)), b"")
def do_http(method, url, **kwargs):
response = yield treq.request(method, url, persistent=False, **kwargs)
body = yield treq.content(response)
# TODO: replace this with response.fail_for_status when
# https://github.com/twisted/treq/pull/159 has landed
if 400 <= response.code < 600:
raise Error(response.code, response=body)
defer.returnValue(body)
def test_content(self):
d = content(self.response)
self.protocol.dataReceived('foo')
self.protocol.dataReceived('bar')
self.protocol.connectionLost(Failure(ResponseDone()))
self.assertEqual(self.successResultOf(d), 'foobar')
def got_result(result):
if result.code in success_codes:
action.addSuccessFields(response_code=result.code)
return json_content(result)
else:
d = content(result)
d.addCallback(error, result.code)
return d
def test_delete_server(self):
"""
Test to verify :func:`delete_server` on ``DELETE /v2.0/<tenant_id>/servers/<server_id>``
"""
delete_server = request(self, self.root, "DELETE", self.uri + '/servers/' + self.server_id)
delete_server_response = self.successResultOf(delete_server)
self.assertEqual(delete_server_response.code, 204)
self.assertEqual(self.successResultOf(treq.content(delete_server_response)),
b"")
def test_content_cached(self):
d1 = content(self.response)
self.protocol.dataReceived('foo')
self.protocol.dataReceived('bar')
self.protocol.connectionLost(Failure(ResponseDone()))
self.assertEqual(self.successResultOf(d1), 'foobar')
def _fail_deliverBody(protocol):
self.fail("deliverBody unexpectedly called.")
self.response.original.deliverBody.side_effect = _fail_deliverBody
d3 = content(self.response)
self.assertEqual(self.successResultOf(d3), 'foobar')
self.assertNotIdentical(d1, d3)
def search(self, name):
exception = self.master.modules["commands"].exception
u = yield self.config.get("username")
p = yield self.config.get("password")
r = yield treq.get("http://myanimelist.net/api/anime/search.xml", params={"q": name}, auth=(u,p))
d = yield treq.content(r)
s = BeautifulSoup(d)
if not s.anime:
raise exception(u"Could not find \"{}\" on MAL".format(name))
returnValue((int(s.anime.entry.id.string), s.anime.entry.title.string))
def __resp2Json(self, a_response):
# We can't use treq.json_content() here because we want an
# opportunity to recover from the content received if we can't
# parse any JSON; we also want to keep the response around, which
# is why we don't want to just add treq.content() as a callback
# directly to treq.request()
d = treq.content(a_response)
d.addCallback(self.__content2Json, a_response)
return d
def got_response(response):
if response.code in success_codes:
action.addSuccessFields(response_code=response.code)
d = json_content(response)
d.addCallback(lambda decoded_body: (decoded_body, response.headers))
return d
else:
d = content(response)
d.addCallback(error, response.code)
return d
def characters(self, mal_id, mal_title):
l = []
# MAL has some stupid DDOS protection bullshit, so use google instead
q = u"cache:myanimelist.net/anime/{:d}/{}/characters".format(mal_id, mal_title.replace(" ", "_"))
r = yield treq.get("http://webcache.googleusercontent.com/search", params={"q":q})
d = yield treq.content(r)
s = BeautifulSoup(d, "html5lib")
for c in s.find_all("a", href=re.compile(r"^/character/\d+")):
if c.string:
l.append(u" ".join(reversed(c.string.split(u", "))))
returnValue(l)
def slask(request):
monits = []
for host in list(_monithosts):
try:
monitrequest = yield treq.get('%s/_status?format=xml' % host, auth=('admin', 'monit'), timeout=0.5)
except ConnectionRefusedError as err:
continue
body = yield treq.content(monitrequest)
monits.append(objectify.fromstring(body))
defer.returnValue(objectJSONEncoder().encode(monits))
def got_response(response):
result = content(response)
# Gear can return a variety of 2xx success codes:
if response.code // 100 == 2:
result.addCallback(lambda _: True)
elif response.code == NOT_FOUND:
result.addCallback(lambda _: False)
else:
result.addCallback(
lambda data: fail(GearError(response.code, data)))
return result
def assertVerified(self, ip, challenge, response):
params = {
'privatekey': self.private_key,
'remoteip': ip,
'challenge': challenge,
'response': response,
}
response = yield treq.post('http://www.google.com/recaptcha/api/verify',
params)
content = yield treq.content(response)
if not content.startswith('true\n'):
raise Exception('Captcha failed')
def _bulk_delete(test_case, root, uri, lb_id, node_ids):
"""Bulk delete multiple nodes."""
query = '?' + '&'.join('id=' + str(node_id) for node_id in node_ids)
endpoint = uri + '/loadbalancers/' + str(lb_id) + '/nodes' + query
d = request(test_case, root, "DELETE", endpoint)
response = test_case.successResultOf(d)
body = test_case.successResultOf(treq.content(response))
if body == '':
body = EMPTY_RESPONSE
else:
body = json.loads(body)
return response, body
def _handle_error_response(self, response):
def _raise_error(content_str):
content_str = content_str.strip()
if len(content_str) > 0:
raise MarconiError(json.loads(content_str))
else:
msg = 'Received {code} response with empty body'.format(code=response.code)
raise MarconiError(msg)
d = content(response)
d.addCallback(_raise_error)
return d
def put_container(self, container_path=None, expected_result=201):
"""
PUT a container - create a container
"""
container_uri = (
container_path if container_path is not None else self.uri)
create_container = request(self, self.root, b"PUT", container_uri)
create_container_response = self.successResultOf(create_container)
self.assertEqual(create_container_response.code, expected_result)
self.assertEqual(
self.successResultOf(treq.content(create_container_response)),
b"",
)
def GetLatestVersion(self):
try:
r = yield treq.get("http://download.longaccess.com/latest.json")
if r.code != 200:
raise Exception("error getting latest version: {} {}".format(
r.code, r.phrase))
r = yield treq.content(r)
vinfo = {k: v for k, v in json.loads(r).iteritems()
if k in ("version", "description", "uri")}
except Exception:
getLogger().debug("couldn't get latest version", exc_info=True)
vinfo = {"version": __version__}
defer.returnValue(ttypes.VersionInfo(**vinfo))