def setUp(self):
"""
Create a :obj:`MimicCore` with :obj:`LoadBalancerApi` as the only plugin.
And create a load balancer and add nodes to the load balancer.
"""
helper = APIMockHelper(self, [LoadBalancerApi()])
self.root = helper.root
self.uri = helper.uri
create_lb = request(
self, self.root, "POST", self.uri + '/loadbalancers',
json.dumps({
"loadBalancer": {
"name": "test_lb",
"protocol": "HTTP",
"virtualIps": [{"type": "PUBLIC"}]
}
})
)
create_lb_response = self.successResultOf(create_lb)
self.create_lb_response_body = self.successResultOf(treq.json_content(
create_lb_response))
create_node = request(
self, self.root, "POST", self.uri + '/loadbalancers/' +
str(self.create_lb_response_body["loadBalancer"]["id"]) + '/nodes',
json.dumps({"nodes": [{"address": "127.0.0.1",
"port": 80,
"condition": "ENABLED",
"type": "PRIMARY",
"weight": 10}]})
)
self.create_node_response = self.successResultOf(create_node)
self.create_node_response_body = self.successResultOf(treq.json_content(
self.create_node_response))
self.node = self.create_node_response_body["nodes"]
def test_server_in_building_state_for_specified_time(self):
"""
Test to verify :func:`create_server` creates a server in BUILD
status for the time specified in the metadata.
"""
metadata = {"server_building": 1}
# create server with metadata to keep the server in building state for
# 3 seconds
create_server_response = self.create_server(metadata=metadata)
# verify the create server was successful
self.assertEquals(create_server_response.code, 202)
create_server_response_body = self.successResultOf(
treq.json_content(create_server_response))
# get server and verify status is BUILD
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
get_server_response_body = self.successResultOf(
treq.json_content(get_server_response))
self.assertEquals(get_server_response_body['server']['status'], "BUILD")
# Time Passes...
self.helper.clock.advance(2.0)
# get server and verify status changed to active
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
get_server_response_body = self.successResultOf(
treq.json_content(get_server_response))
self.assertEquals(get_server_response_body['server']['status'], "ACTIVE")
def _on_response(response):
if response.code == 201:
json_content(response).addCallback(_construct_message, response)
elif response.code == 204:
reactor.callLater(polling_interval, _perform_call)
else:
return self._handle_error_response(response)
def lookupCharacters(member, characters):
response = yield treq.get("http://www.bungie.net/Platform/User/GetBungieAccount/" + member["membershipId"].encode("UTF-8") + "/254/")
data = yield treq.json_content(response)
for account in data["Response"]["destinyAccounts"]:
if account["userInfo"]["membershipType"] == 1:
platform = "xbox"
elif account["userInfo"]["membershipType"] == 2:
platform = "playstation"
else:
continue
for character in account["characters"]:
character_data = {
"bungieId": member["membershipId"],
"accountId": account["userInfo"]["membershipId"],
"characterId": character["characterId"],
"name": account["userInfo"]["displayName"],
"race": character["race"]["raceName"],
"gender": character["gender"]["genderName"],
"class": character["characterClass"]["className"],
"level": character["level"],
"levelString": "{:,d}".format(character["level"]),
"icon": "http://bungie.net" + character["emblemPath"],
"background": "http://bungie.net" + character["backgroundPath"],
# Default values for extra data (in case it fails)
"light": 0,
"lightString": "{:,d}".format(0),
"grimoire": 0,
"grimoireString": "{:,d}".format(0),
"minutesPlayed": 0,
"minutesPlayedString": "{:,d}".format(0),
"lastSeen": "",
"lastSeenString": ""
}
character_data["style"] = 'background: url("' + character_data["background"] + '")'
try:
response = yield treq.get("http://www.bungie.net/Platform/Destiny/{!s}/Account/{!s}/Character/{!s}/".format(account["userInfo"]["membershipType"], account["userInfo"]["membershipId"], character["characterId"]))
extra_data = yield treq.json_content(response)
extra_data = extra_data["Response"]["data"]["characterBase"]
character_data["light"] = extra_data["stats"]["STAT_LIGHT"]["value"]
character_data["lightString"] = "{:,d}".format(character_data["light"])
character_data["grimoire"] = extra_data["grimoireScore"]
character_data["grimoireString"] = "{:,d}".format(character_data["grimoire"])
character_data["minutesPlayed"] = int(extra_data["minutesPlayedTotal"])
character_data["minutesPlayedString"] = "{:,d}".format(character_data["minutesPlayed"])
character_data["lastSeen"] = extra_data["dateLastPlayed"]
character_data["lastSeenString"] = datetime.datetime.strptime(extra_data["dateLastPlayed"], "%Y-%m-%dT%H:%M:%SZ").strftime("%B %d, %I:%M%p")
except:
pass
characters[platform].append(character_data)
def check(self):
# Hax
try:
irc = self.master.modules["irc"]
words = yield self.config.get("words", [])
response = yield treq.get("http://a.4cdn.org/a/threads.json")
threads = yield treq.json_content(response)
nthreads, check = {}, []
for o in threads:
for t in o["threads"]:
nthreads[t["no"]] = t["last_modified"]
if t["no"] not in self.threads:
self.threads[t["no"]] = 0
if t["last_modified"] > self.threads[t["no"]]:
check.append(t["no"])
if not self.initial:
threads = []
for t in check:
response = yield treq.get("http://a.4cdn.org/a/res/{:d}.json".format(t))
if response.code == 200:
data = yield treq.json_content(response)
found, posts = set(), []
for p in data["posts"]:
if p["time"] > self.threads[t] and "com" in p:
f = set(filter(lambda x: re.search(r"\b" + x.lower() + r"\b", p["com"].lower()), words))
if f:
found.update(f)
posts.append((p["no"], p["com"]))
if found and posts:
threads.append((t, found, posts))
if len(threads) < 5:
for t, found, posts in threads:
p, also = posts[0], (u"(also {} in same thread)".format(", ".join([str(x[0]) for x in posts[1:]])) if posts[1:] else u"")
url = "https://archive.foolz.us/a/thread/{:d}/#{:d}".format(t, p[0])
#response = yield treq.post("https://www.googleapis.com/urlshortener/v1/url?key="+API_KEY, json.dumps({"longUrl": url}), headers={'Content-Type': ['application/json']})
#data = yield treq.json_content(response)
#url = data["id"]
excerpt = p[1].replace("<br>", "\n")
excerpt = BeautifulSoup(excerpt).get_text()
excerpt = re.sub("\s+", " ", excerpt)
excerpt = excerpt if len(excerpt) <= 100 else excerpt[:97]+"..."
irc.msg(u"#commie-subs", u"\u00039>{} mentioned on /a/ at {} [{}] {}".format(u", ".join(found), url, excerpt, also))
else:
t, found = [str(x[0]) for x in threads], reduce(lambda a,b: a|b, [x[1] for x in threads])
irc.msg(u"#commie-subs", u"\u00039>{} flooded /a/ in threads {}".format(u", ".join(found), u", ".join(t)))
self.threads = nthreads
self.initial = False
except:
self.err(u"4chan module broke horribly :(")
def decode_or_error(self, response, cb, *args, **kwargs):
d = treq.json_content(response)
d.addCallback(cb, *args, **kwargs)
errcb = lambda err, *args, **kwargs: print(
"Error when calling %s %s: %s" % (' '.join(map(str, args)),
kwargs, err.getBriefTraceback()))
d.addErrback(errcb, *args, **kwargs)
def test_json_content(self):
d = json_content(self.response)
self.protocol.dataReceived('{"msg":"hello!"}')
self.protocol.connectionLost(Failure(ResponseDone()))
self.assertEqual(self.successResultOf(d), {"msg": "hello!"})
def setUp(self):
"""
Create a :obj:`MimicCore` with :obj:`LoadBalancerApi` as the only plugin.
And create a load balancer and add nodes to the load balancer.
"""
helper = APIMockHelper(self, [LoadBalancerApi()])
self.root = helper.root
self.uri = helper.uri
create_lb = request(
self, self.root, "POST", self.uri + '/loadbalancers',
json.dumps({
"loadBalancer": {
"name": "test_lb",
"protocol": "HTTP",
"virtualIps": [{"type": "PUBLIC"}]
}
})
)
create_lb_response = self.successResultOf(create_lb)
self.create_lb_response_body = self.successResultOf(treq.json_content(
create_lb_response))
self.lb_id = self.create_lb_response_body["loadBalancer"]["id"]
create_node = self._create_nodes(["127.0.0.1"])
[(self.create_node_response, self.create_node_response_body)] = create_node
self.node = self.create_node_response_body["nodes"]
def test_add_load_balancer_with_nodes(self):
"""
Test to verify :func:`add_load_balancer` on ``POST /v1.0/<tenant_id>/loadbalancers``,
with nodes
"""
lb_name = 'mimic_lb'
create_lb = request(
self, self.root, "POST", self.uri + '/loadbalancers',
json.dumps({
"loadBalancer": {
"name": lb_name,
"protocol": "HTTP",
"virtualIps": [{"type": "PUBLIC"}],
"nodes": [{"address": "127.0.0.2",
"port": 80,
"condition": "ENABLED",
"type": "PRIMARY"},
{"address": "127.0.0.0",
"port": 80,
"condition": "ENABLED",
"type": "SECONDARY"}]
}
})
)
create_lb_response = self.successResultOf(create_lb)
create_lb_response_body = self.successResultOf(treq.json_content(create_lb_response))
self.assertEqual(create_lb_response.code, 202)
self.assertEqual(len(create_lb_response_body['loadBalancer']['nodes']), 2)
def test_load_balancer_goes_into_error_state_when_adding_node(self):
"""
Test to verify a load balancer goes into error state when adding a node.
Adding a node to a loadbalancer in ERROR state results in 422.
And such a load balancer can only be deleted.
"""
metadata = [{"key": "lb_error_state", "value": "error"}]
create_response = self._create_loadbalancer_for_given_metadata(metadata)
self.assertEqual(create_response.code, 202)
create_lb_response_body = self.successResultOf(treq.json_content(create_response))
lb = create_lb_response_body["loadBalancer"]
self.assertEqual(lb["status"], "ACTIVE")
create_node_response = self._add_node_to_lb(lb["id"])
self.assertEqual(create_node_response.code, 200)
# get loadbalncer after adding node and verify its in error state
errored_lb = self._get_loadbalancer(lb["id"])
self.assertEqual(errored_lb["loadBalancer"]["status"], "ERROR")
# adding another node to a lb in ERROR state, results in 422
create_node_response = self._add_node_to_lb(lb["id"])
self.assertEqual(create_node_response.code, 422)
# An lb in ERROR state can be deleted
delete_lb = request(self, self.root, "DELETE", self.uri + '/loadbalancers/' +
str(lb["id"]))
delete_lb_response = self.successResultOf(delete_lb)
self.assertEqual(delete_lb_response.code, 202)
def got_result(result):
if result.code == success_code:
return json_content(result)
else:
d = content(result)
d.addCallback(error, result.code)
return d
def test_basic_auth(self):
response = yield self.get("/basic-auth/treq/treq", auth=("treq", "treq"))
self.assertEqual(response.code, 200)
yield print_response(response)
json = yield treq.json_content(response)
self.assertTrue(json["authenticated"])
self.assertEqual(json["user"], "treq")
def request_done(response):
def _raise_error(body):
exceptions.raise_exceptions(response.code, body)
return body
return treq.json_content(response.original).addCallback(
_raise_error)
def test_get_server_ips(self):
"""
Test to verify :func:`get_ips` on ``GET /v2.0/<tenant_id>/servers/<server_id>/ips``
"""
get_server_ips = request(self, self.root, "GET",
self.uri + '/servers/' + self.server_id + '/ips')
get_server_ips_response = self.successResultOf(get_server_ips)
get_server_ips_response_body = self.successResultOf(
treq.json_content(get_server_ips_response))
self.assertEqual(get_server_ips_response.code, 200)
list_servers_detail = request(self, self.root, "GET", self.uri + '/servers/detail')
list_servers_detail_response = self.successResultOf(list_servers_detail)
list_servers_detail_response_body = self.successResultOf(
treq.json_content(list_servers_detail_response))
self.assertEqual(get_server_ips_response_body['addresses'],
list_servers_detail_response_body['servers'][0]['addresses'])
def _update_ticket(self, _id, _rev, data):
"""
Update a ticket in CouchDB.
"""
data[u'expires'] = data[u'expires'].strftime('%Y-%m-%dT%H:%M:%S')
if u'pgts' in data:
data[u'pgts'] = list(data[u'pgts'])
url = '''%(scheme)s%(host)s:%(port)s/%(db)s/%(docid)s''' % {
'scheme': self._scheme,
'host': self._couch_host,
'port': self._couch_port,
'db': self._couch_db,
'docid': _id}
url = url.encode('utf-8')
data['_rev'] = _rev.encode('utf-8')
try:
doc = json.dumps(data)
except Exception as ex:
self.debug("[DEBUG][CouchDB] Failed to serialze doc:\n%s" % (str(data)))
raise
httpClient = self.httpClientFactory(self.reactor)
self.debug('''[DEBUG][CouchDB] request_method="PUT" url="{0}"'''.format(url))
self.debug('''[DEBUG][CouchDB] document => {0}'''.format(data))
response = yield httpClient.put(
url,
data=doc,
auth=(self._couch_user, self._couch_passwd),
headers=Headers({
'Accept': ['application/json'],
'Content-Type': ['application/json']}))
response = yield http_status_filter(response, [(201,201)], CouchDBError)
doc = yield treq.json_content(response)
defer.returnValue(None)
def test_delete(self):
response = yield self.delete("/delete")
self.assertEqual(response.code, OK)
data = yield treq.json_content(response)
self.assertIsInstance(data, dict)
self.assertEqual(data["headers"]["User-Agent"], USERAGENT)
self.assertIsNone(data["json"])
def lookupClan(name):
response = yield treq.post("http://www.bungie.net/Platform/Group/Search/", data=json.dumps({
"contents": {
"searchValue": name
},
"currentPage": 1,
"itemsPerPage": 1
}))
data = yield treq.json_content(response)
if data["Response"]["results"]:
clan = data["Response"]["results"][0]["detail"]
if clan["memberCount"] > 1000:
returnValue({
"id": 0,
"name": '"{}" is too big'.format(clan["name"]),
"motto": "Clans with over 1,000 members can't be processed",
})
else:
returnValue({
"id": int(clan["groupId"]),
"name": clan["name"],
"motto": HTMLParser.HTMLParser().unescape(clan["about"]),
})
else:
returnValue({
"id": 0,
"name": "No Clan Found",
"motto": "Better luck next time"
})
def setUp(self):
"""
Create a :obj:`MimicCore` with :obj:`QueueApi` as the only plugin,
and create a queue
"""
self.clock = Clock()
self.core = MimicCore(self.clock, [QueueApi()])
self.root = MimicRoot(self.core).app.resource()
self.response = request(
self, self.root, b"POST", "/identity/v2.0/tokens",
json.dumps({
"auth": {
"passwordCredentials": {
"username": "******",
"password": "******",
},
}
}).encode("utf-8")
)
self.auth_response = self.successResultOf(self.response)
self.json_body = self.successResultOf(
treq.json_content(self.auth_response))
self.uri = self.json_body['access']['serviceCatalog'][0]['endpoints'][0]['publicURL']
self.queue_name = "test_queue"
self.create_queue = request(
self, self.root, b"PUT", self.uri + '/queues/' + self.queue_name)
self.create_queue_response = self.successResultOf(self.create_queue)
def request_done(response):
global req_made
deferred = treq.json_content(response)
req_made += 1
deferred.addCallback(body_received)
deferred.addErrback(lambda x: None) # ignore errors
return deferred
def set(self, key, value, lease=None, prev_kv=None):
"""
Put puts the given key into the key-value store.
A put request increments the revision of the key-value
store and generates one event in the event history.
URL: /v3alpha/kv/put
:param key: key is the key, in bytes, to put into the key-value store.
:type key: bytes
:param lease: lease is the lease ID to associate with the key in the key-value store. A lease\nvalue of 0 indicates no lease.
:type lease: int
:param prev_kv: If prev_kv is set, etcd gets the previous key-value pair before changing it.\nThe previous key-value pair will be returned in the put response.
:type prev_kv: bool
:param value: value is the value, in bytes, to associate with the key in the key-value store.
:key value: bytes
"""
url = u'{}/v3alpha/kv/put'.format(self._url).encode()
obj = {
u'key': binascii.b2a_base64(key).decode(),
u'value': binascii.b2a_base64(value).decode()
}
data = json.dumps(obj).encode('utf8')
response = yield treq.post(url, data, headers=self.REQ_HEADERS)
obj = yield treq.json_content(response)
revision = obj[u'header'][u'revision']
returnValue(revision)
def test_load_balancer_goes_into_pending_update_state(self):
"""
Test to verify a load balancer goes into PENDING-UPDATE state, for
the given time in seconds when any action other than DELETE is performed
on the lb.
Adding a node to a loadbalancer in PENDING-UPDATE state results in 422.
And such a load balancer can be deleted.
"""
metadata = [{"key": "lb_pending_update", "value": 30}]
create_response = self._create_loadbalancer_for_given_metadata(metadata)
self.assertEqual(create_response.code, 202)
create_lb_response_body = self.successResultOf(treq.json_content(create_response))
lb = create_lb_response_body["loadBalancer"]
self.assertEqual(lb["status"], "ACTIVE")
create_node_response = self._add_node_to_lb(lb["id"])
self.assertEqual(create_node_response.code, 200)
# get loadbalncer after adding node and verify its in PENDING-UPDATE state
errored_lb = self._get_loadbalancer(lb["id"])
self.assertEqual(errored_lb["loadBalancer"]["status"], "PENDING-UPDATE")
# Trying to add/list/delete node on a lb in PENDING-UPDATE state, results in 422
create_node_response = self._add_node_to_lb(lb["id"])
self.assertEqual(create_node_response.code, 422)
delete_nodes = request(
self, self.root, "DELETE", self.uri + '/loadbalancers/' +
str(lb["id"]) + '/nodes/123')
self.assertEqual(self.successResultOf(delete_nodes).code, 422)
# An lb in PENDING-UPDATE state can be deleted
delete_lb = request(self, self.root, "DELETE", self.uri + '/loadbalancers/' +
str(lb["id"]))
delete_lb_response = self.successResultOf(delete_lb)
self.assertEqual(delete_lb_response.code, 202)
def test_get_container(self):
"""
Creating a container and immediately retrieving it yields an empty list
(since there are no objects) and several headers indicating that no
objects are in the container and they consume no space.
"""
self.createSwiftService()
# create a container
uri = (self.json_body['access']['serviceCatalog'][0]['endpoints'][0]
['publicURL'] + '/testcontainer').encode("ascii")
create_container = request(self, self.root, b"PUT", uri)
self.successResultOf(create_container)
container_response = self.successResultOf(
request(self, self.root, b"GET", uri)
)
self.assertEqual(container_response.code, 200)
container_contents = self.successResultOf(
treq.json_content(container_response)
)
self.assertEqual(container_contents, [])
self.assertEqual(
container_response.headers.getRawHeaders(
b"X-Container-Object-Count")[0], b"0"
)
self.assertEqual(
container_response.headers.getRawHeaders(
b"X-Container-Bytes-Used")[0], b"0"
)
def delete(self, key, range_end=None, prev_kv=False):
"""
Delete value(s) from etcd.
URL: /v3alpha/kv/deleterange
:param key: key is the first key to delete in the range.
:type key: bytes
:param range_end: range_end is the key following the last key to delete
for the range [key, range_end).\nIf range_end is not given, the range
is defined to contain only the key argument.\nIf range_end is one bit
larger than the given key, then the range is all keys with the prefix
(the given key).\nIf range_end is '\\0', the range is all keys greater
than or equal to the key argument.
:key range_end: bytes
:param prev_kv: If prev_kv is set, etcd gets the previous key-value pairs
before deleting it.\nThe previous key-value pairs will be returned in the
delete response.
:key prev_kv: bool
"""
url = u'{}/v3alpha/kv/deleterange'.format(self._url).encode()
obj = {
u'key': binascii.b2a_base64(key).decode(),
u'range_end': binascii.b2a_base64(range_end).decode() if range_end else None,
u'prev_kv': prev_kv
}
data = json.dumps(obj).encode('utf8')
response = yield treq.post(url, data, headers=self.REQ_HEADERS)
obj = yield treq.json_content(response)
res = Deleted.parse(obj)
returnValue(res)
def get_issue_subject(ticket_url, api_key=None):
"""
Find the "subject" string in the JSON data of a ticket.
:param ticket_url: URL for a ticket
:param api_key: API secret key (string), or None if you do not want to
authenticate to Redmine.
:returns: twisted.internet.defere.Deferred. When this Deferred fires, it
will return a "subject" string to its callback.
"""
api_url = "%s.json" % ticket_url
request_headers = {}
if api_key is not None:
request_headers['X-Redmine-API-Key'] = api_key
try:
response = yield treq.get(api_url, headers=request_headers, timeout=5)
if response.code != 200:
subject = 'could not read subject, HTTP code %i' % response.code
defer.returnValue((ticket_url, subject))
else:
content = yield treq.json_content(response)
defer.returnValue((ticket_url, content['issue']['subject']))
except Exception, e:
# For example, if treq.get() timed out, or if treq.json_content() could
# not parse the JSON, etc.
subject = 'could not read subject, %s' % e.message
defer.returnValue((ticket_url, subject))
def lookupMembers(id):
page = 1
hasMore = True if id else False
members = []
characters = {
"playstation": [],
"xbox": []
}
deferreds = []
while hasMore:
# No idea what is different between V1, V2, and V3...
response = yield treq.get("http://www.bungie.net/Platform/Group/{!s}/MembersV3/".format(id), params={
"itemsPerPage": 50,
"currentPage": page
})
data = yield treq.json_content(response)
page += 1
hasMore = data["Response"]["hasMore"]
members.extend(data["Response"]["results"])
# Load character data in parallel
for member in members:
deferreds.append(lookupCharacters(member, characters))
yield DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
for platform_characters in characters.values():
platform_characters.sort(key=lambda c: (c["level"], c["light"]), reverse=True)
returnValue(characters)
def test_delete_server_fails_specified_number_of_times(self):
"""
Test to verify :func: `delete_server` does not delete the server,
and returns the given response code, the number of times specified
in the metadata
"""
deletefail = {"times": 1, "code": 500}
metadata = {"delete_server_failure": json.dumps(deletefail)}
# create server and verify it was successful
create_server_response = self.create_server(metadata=metadata)
self.assertEquals(create_server_response.code, 202)
create_server_response_body = self.successResultOf(
treq.json_content(create_server_response))
# delete server and verify the response
delete_server = request(self, self.root, "DELETE", self.uri + '/servers/'
+ create_server_response_body["server"]["id"])
delete_server_response = self.successResultOf(delete_server)
self.assertEqual(delete_server_response.code, 500)
# get server and verify the server was not deleted
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
self.assertEquals(get_server_response.code, 200)
# delete server again and verify the response
delete_server = request(self, self.root, "DELETE", self.uri + '/servers/'
+ create_server_response_body["server"]["id"])
delete_server_response = self.successResultOf(delete_server)
self.assertEqual(delete_server_response.code, 204)
self.assertEqual(self.successResultOf(treq.content(delete_server_response)),
b"")
# get server and verify the server was deleted this time
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
self.assertEquals(get_server_response.code, 404)
def _fetch_ticket(self, ticket):
"""
Fetch a ticket representation from CouchDB.
"""
url = '''%(scheme)s%(host)s:%(port)s/%(db)s/_design/views/_view/get_ticket''' % {
'scheme': self._scheme,
'host': self._couch_host,
'port': self._couch_port,
'db': self._couch_db}
url = url.encode('utf-8')
params = {'key': json.dumps(ticket.encode('utf-8'))}
self.debug("[DEBUG][CouchDB] _fetch_ticket(), url: %s" % url)
self.debug("[DEBUG][CouchDB] _fetch_ticket(), params: %s" % str(params))
httpClient = self.httpClientFactory(self.reactor)
response = yield httpClient.get(url,
params=params,
headers=Headers({'Accept': ['application/json']}),
auth=(self._couch_user, self._couch_passwd))
response = yield http_status_filter(response, [(200,200)], CouchDBError)
doc = yield treq.json_content(response)
rows = doc[u'rows']
if len(rows) > 0:
entry = rows[0][u'value']
entry[u'expires'] = parse_date(entry[u'expires'])
if u'pgts' in entry:
entry[u'pgts'] = set(entry[u'pgts'])
defer.returnValue(entry)
defer.returnValue(None)
def lookup_timezone(loc, api_key=''):
"""
Determine the timezone of a lat/long pair.
@param loc: lat/long coordinates of a location
@type loc: dict
@type api_key: str
@rtype: defer.Deferred yielding a second offset representing the timezone
"""
try:
res = yield treq.get(("https://maps.googleapis.com/maps/api/timezone/json"),
params={'location': str(loc['lat']) + ',' + str(loc['lng']),
'timestamp': str(now_in_utc_secs()), 'sensor': 'false', 'key': api_key})
if res and res.code == 200:
data = yield treq.json_content(res)
if data['status'] == 'OK':
# API returned timezone info. What we care about: rawOffset
defer.returnValue(int(data['rawOffset'])+int(data['dstOffset']))
else:
log.warn("Bad status response from Google Geocode API: %s" % (data['status']))
elif res is not None:
log.warn("Bad HTTP status from Google Timezone API: %s" % (res.code))
except TypeError:
log.warn("Bad lat/long parameter passed to lookup_timezone: %s" % (loc), exc_info=True)
def lookup_geocode(location, api_key=''):
"""
Determine the lat/long coordinates for a location name.
:param location: location name
:type location: str
:type api_key: str
:return: a dict with keys 'lat', 'lng', 'loc' that contain
the lat/long coordinates and placename for the location.
:rtype: defer.Deferred
"""
try:
res = yield treq.get("http://maps.googleapis.com/maps/api/geocode/json",
params={'address': location, 'sensor': 'false', 'key': api_key})
if res and res.code == 200:
data = yield treq.json_content(res)
if data['status'] == 'OK':
# API returned at least one geocode
ret = data['results'][0]['geometry']['location']
ret['loc'] = data['results'][0]['formatted_address']
defer.returnValue(ret)
else:
log.warn("Bad status response from Google Geocode API: %s" % (data['status']))
elif res is not None:
log.warn("Bad HTTP status from Google Geocode API: %s" % (res.code))
except TypeError:
log.warn("Bad location passed to lookup_geocode", exc_info=True)
def test_get_server_limits(self):
"""
Test to verify :func:`get_limit` on ``GET /v2.0/<tenant_id>/limits``
"""
get_server_limits = request(self, self.root, "GET", self.uri + '/limits')
get_server_limits_response = self.successResultOf(get_server_limits)
self.assertEqual(get_server_limits_response.code, 200)
self.assertTrue(self.successResultOf(treq.json_content(get_server_limits_response)))
def test_create_queue(self):
"""
Test to verify :func:`add queue` on ``PUT /v2.0/<tenant_id>/queues/<queue_name>``
"""
self.create_queue_body = self.successResultOf(
treq.json_content(self.create_queue_response))
self.assertEqual(self.create_queue_response.code, 201)
self.assertTrue(str(self.create_queue_body), 'null')
def got_result(result):
if result.code in success_codes:
action.addSuccessFields(response_code=result.code)
return json_content(result)
else:
d = content(result)
d.addCallback(error, result.code)
return d
def add_product_sales_quantity(self, market_hash):
url = self.url + '/product/v1/product/sales_quantity/add/'
payload = {'market_hash': market_hash}
header = {"MARKET-KEY": self.public_key, "MARKET-TOKEN": self.token,
'Content-Type': 'application/json'}
resp = yield treq.post(url, headers=header, json=payload)
confirm_info = yield treq.json_content(resp)
return confirm_info
def query_promotion(self):
url = self.url + 'product/v1/recommend_product/list/'
header = {'Content-Type': 'application/json', 'MARKET-KEY': self.public_key,
'MARKET-TOKEN': self.token}
resp = yield treq.get(url=url, headers=header)
confirm_info = yield treq.json_content(resp)
logger.debug("promotion: %s", confirm_info)
return confirm_info['data']
def query_by_tag(self, tag):
url = self.url + 'product/v1/es_product/search/?status=0&tag=' + str(tag)
header = {"MARKET-KEY": self.public_key, "MARKET-TOKEN": self.token,
'Content-Type': 'application/json'}
resp = yield treq.get(url, headers=header)
confirm_info = yield treq.json_content(resp)
logger.debug('query by tag confirm info: %s', confirm_info)
return confirm_info['results']
def test_json_content(self):
self.response.headers = Headers()
d = json_content(self.response)
self.protocol.dataReceived(b'{"msg":"hello!"}')
self.protocol.connectionLost(Failure(ResponseDone()))
self.assertEqual(self.successResultOf(d), {"msg": "hello!"})
def unsubscribe_seller(self, seller_pub_key):
url = self.url + '/product/v1/product/seller/unsubscribe/'
payload = {'public_key': self.public_key, 'seller_public_key': seller_pub_key}
header = {"MARKET-KEY": self.public_key, "MARKET-TOKEN": self.token,
'Content-Type': 'application/json'}
resp = yield treq.post(url, headers=header, json=payload)
confirm_info = yield treq.json_content(resp)
return confirm_info['status']
def test_basic_auth(self):
response = yield self.get('/basic-auth/treq/treq',
auth=('treq', 'treq'))
self.assertEqual(response.code, 200)
yield print_response(response)
json = yield treq.json_content(response)
self.assertTrue(json['authenticated'])
self.assertEqual(json['user'], 'treq')
def got_result(result):
if result.code in success_codes:
action.addSuccessFields(response_code=result.code)
return json_content(result)
else:
d = content(result)
d.addCallback(error, result.code)
return d
def process_response(self, response):
"""
Receives a treq response, collects the content, and attempts to smartly decode
the content based on the response headers.
If headers don't match, a YomboWarning will be raised.
:param response:
:param headers:
:return:
"""
raw_content = yield treq.content(response)
content = raw_content
headers = self.clean_headers(response, True)
if HEADER_CONTENT_TYPE in headers:
content_type = headers[HEADER_CONTENT_TYPE][0]
else:
content_type = None
if content_type == HEADER_CONTENT_TYPE:
try:
content = yield treq.json_content(response)
content_type = "dict"
except Exception as e:
raise YomboWarning(f"Receive response reported json, but found an error: {e}")
elif content_type == CONTENT_TYPE_MSGPACK:
try:
content = msgpack.loads(raw_content)
except Exception:
if len(content) == 0:
return "dict", {}
raise YomboWarning(f"Receive response reported msgpack, but isn't: {content}")
else:
content_type = "string"
try:
content = json.loads(raw_content)
content_type = "dict"
except Exception:
try:
content = msgpack.loads(raw_content)
content_type = "dict"
except Exception:
content = raw_content
content = bytes_to_unicode(content)
# return {
# "content_type": content_type,
# "headers": response.all_headers,
# }
self.content = bytes_to_unicode(content)
self.content_raw = raw_content
self.request = response.request.original
self.response = response
self.content_type = content_type
self.response_phrase = bytes_to_unicode(response.phrase)
self.response_code = response.code
self.headers = headers
def isRegistered(self, public_key):
header = {'Content-Type': 'application/json'}
data = {'public_key': public_key}
resp = yield treq.post(url=self.url + 'account/v1/isregistered/',
headers=header,
json=data,
persistent=False)
result = yield treq.json_content(resp)
return result['status']
def query_product(self, keyword):
logger.debug('keywords: %s', keyword)
header = {'Content-Type': 'application/json'}
url = self.url + 'product/v1/es_product/search/?search=' + keyword
logger.debug('query url: %s', url)
resp = yield treq.get(url=url, headers=header, persistent=False)
confirm_info = yield treq.json_content(resp)
logger.debug("query product confirm info: %s", confirm_info)
return confirm_info['results']
def query_hot_tag(self):
url = self.url + 'main/v1/hot_tag/list/'
header = {'Content-Type': 'application/json', 'MARKET-KEY': self.public_key,
'MARKET-TOKEN': self.token}
resp = yield treq.get(url=url, headers=header)
confirm_info = yield treq.json_content(resp)
print(confirm_info)
logger.debug("hot tag: %s", confirm_info)
return confirm_info['data']
def test_server_in_error_state(self):
"""
Test to verify :func:`create_server` creates a server in ERROR state.
"""
metadata = {"server_error": 1}
# create server with metadata to set status in ERROR
create_server_response = self.create_server(metadata=metadata)
# verify the create server was successful
self.assertEquals(create_server_response.code, 202)
create_server_response_body = self.successResultOf(
treq.json_content(create_server_response))
# get server and verify status is ERROR
get_server = request(self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
get_server_response_body = self.successResultOf(
treq.json_content(get_server_response))
self.assertEquals(get_server_response_body['server']['status'], "ERROR")
def query_by_seller(self, public_key):
logger.debug("in query by seller")
logger.debug("seller's public key: %s", self.public_key)
logger.debug("public key used by query: %s", public_key)
url = self.url + 'product/v1/es_product/search/?ordering=-created&offset=0&limit=100&status=0&seller=' + str(public_key)
header = {"MARKET-KEY": self.public_key, "MARKET-TOKEN": self.token, 'Content-Type': 'application/json'}
resp = yield treq.get(url, headers=header, persistent=False)
confirm_info = yield treq.json_content(resp)
return confirm_info['results']
def query_username(self, app):
header = {'Content-Type': 'application/json'}
data = {'public_key': self.public_key}
resp = yield treq.post(url=self.url + 'account/v1/username/',
headers=header,
json=data,
persistent=False)
username = yield treq.json_content(resp)
app.username = username['username']
def login(self):
header = {'Content-Type': 'application/json'}
data = {'public_key': self.public_key}
resp = yield treq.post(url=self.url + 'account/v1/login/', headers=header, json=data,
persistent=False)
confirm_info = yield treq.json_content(resp)
logger.debug("login response: %s", confirm_info)
self.nonce = confirm_info['message']
logger.debug('nonce: %s', self.nonce)
signature = ECCipher.create_signature(self.account.private_key, self.nonce)
header_confirm = {'Content-Type': 'application/json'}
data_confirm = {'public_key': self.public_key, 'code': Encoder.bytes_to_hex(signature)}
resp = yield treq.post(self.url + 'account/v1/confirm/', headers=header_confirm,
json=data_confirm,
persistent=False)
confirm_info = yield treq.json_content(resp)
self.token = confirm_info['message']
return confirm_info['status']
def get_json(self,
destination,
path,
args=None,
retry_on_dns_fail=True,
timeout=None,
ignore_backoff=False):
""" GETs some json from the given host homeserver and path
Args:
destination (str): The remote server to send the HTTP request
to.
path (str): The HTTP path.
args (dict|None): A dictionary used to create query strings, defaults to
None.
timeout (int): How long to try (in ms) the destination for before
giving up. None indicates no timeout and that the request will
be retried.
ignore_backoff (bool): true to ignore the historical backoff data
and try the request anyway.
Returns:
Deferred: Succeeds when we get a 2xx HTTP response. The result
will be the decoded JSON body.
Fails with ``HTTPRequestException`` if we get an HTTP response
code >= 300.
Fails with ``NotRetryingDestination`` if we are not yet ready
to retry this server.
Fails with ``FederationDeniedError`` if this destination
is not on our federation whitelist
"""
logger.debug("get_json args: %s", args)
logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)
response = yield self._request(
destination,
"GET",
path,
query=args,
retry_on_dns_fail=retry_on_dns_fail,
timeout=timeout,
ignore_backoff=ignore_backoff,
)
if 200 <= response.code < 300:
# We need to update the transactions table to say it was sent?
check_content_type_is_json(response.headers)
with logcontext.PreserveLoggingContext():
d = treq.json_content(response)
d.addTimeout(self.default_timeout, self.hs.get_reactor())
body = yield make_deferred_yieldable(d)
defer.returnValue(body)
def test_get(self):
"""
happy-path test of a GET request with an explicit port
"""
self.reactor.lookups["testserv"] = "1.2.3.4"
test_d = self._make_get_request(b"matrix://testserv:8448/foo/bar")
# Nothing happened yet
self.assertNoResult(test_d)
# Make sure treq is trying to connect
clients = self.reactor.tcpClients
self.assertEqual(len(clients), 1)
(host, port, client_factory, _timeout, _bindAddress) = clients[0]
self.assertEqual(host, "1.2.3.4")
self.assertEqual(port, 8448)
# make a test server, and wire up the client
http_server = self._make_connection(client_factory,
expected_sni=b"testserv")
self.assertEqual(len(http_server.requests), 1)
request = http_server.requests[0]
self.assertEqual(request.method, b"GET")
self.assertEqual(request.path, b"/foo/bar")
self.assertEqual(request.requestHeaders.getRawHeaders(b"host"),
[b"testserv:8448"])
self.assertEqual(request.requestHeaders.getRawHeaders(b"user-agent"),
[b"test-agent"])
content = request.content.read()
self.assertEqual(content, b"")
# Deferred is still without a result
self.assertNoResult(test_d)
# send the headers
request.responseHeaders.setRawHeaders(b"Content-Type",
[b"application/json"])
request.write("")
self.reactor.pump((0.1, ))
response = self.successResultOf(test_d)
# that should give us a Response object
self.assertEqual(response.code, 200)
# Send the body
request.write('{ "a": 1 }'.encode("ascii"))
request.finish()
self.reactor.pump((0.1, ))
# check it can be read
json = self.successResultOf(treq.json_content(response))
self.assertEqual(json, {"a": 1})
def test_get_server_flavor(self):
"""
Test to verify :func:`get_image` on ``GET /v2.0/<tenant_id>/flavors/<flavor_id>``
"""
get_server_flavor = request(self, self.root, "GET", self.uri + '/flavors/test-flavor-id')
get_server_flavor_response = self.successResultOf(get_server_flavor)
get_server_flavor_response_body = self.successResultOf(
treq.json_content(get_server_flavor_response))
self.assertEqual(get_server_flavor_response.code, 200)
self.assertEqual(get_server_flavor_response_body['flavor']['id'], 'test-flavor-id')
def test_server_in_building_state_for_specified_time(self):
"""
Test to verify :func:`create_server` creates a server in BUILD
status for the time specified in the metadata.
"""
metadata = {"server_building": "1"}
# create server with metadata to keep the server in building state for
# 3 seconds
create_server_response = self.create_server(metadata=metadata)
# verify the create server was successful
self.assertEquals(create_server_response.code, 202)
create_server_response_body = self.successResultOf(
treq.json_content(create_server_response))
# get server and verify status is BUILD
get_server = request(
self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
get_server_response_body = self.successResultOf(
treq.json_content(get_server_response))
self.assertEquals(get_server_response_body['server']['status'],
"BUILD")
# List servers with details and verify the server is in BUILD status
list_servers = request(self, self.root, "GET",
self.uri + '/servers/detail')
list_servers_response = self.successResultOf(list_servers)
self.assertEquals(list_servers_response.code, 200)
list_servers_response_body = self.successResultOf(
treq.json_content(list_servers_response))
self.assertEquals(len(list_servers_response_body['servers']), 1)
building_server = list_servers_response_body['servers'][0]
self.assertEquals(building_server['status'], "BUILD")
# Time Passes...
self.helper.clock.advance(2.0)
# get server and verify status changed to active
get_server = request(
self, self.root, "GET", self.uri + '/servers/' +
create_server_response_body["server"]["id"])
get_server_response = self.successResultOf(get_server)
get_server_response_body = self.successResultOf(
treq.json_content(get_server_response))
self.assertEquals(get_server_response_body['server']['status'],
"ACTIVE")
def query_by_following_seller(self):
header = {"MARKET-KEY": self.public_key, "MARKET-TOKEN": self.token,
'Content-Type': 'application/json'}
url = utils.build_url(self.url + "product/v1/my_seller/product/search/")
logger.debug(url)
resp = yield treq.get(url, headers=header)
logger.debug(resp)
comment_info = yield treq.json_content(resp)
logger.debug('query by following tag confirm: %s', comment_info)
return comment_info['data']
def query_comment_by_hash(self, market_hash):
header = {"MARKET-KEY": self.public_key, "MARKET-TOKEN": self.token,
'Content-Type': 'application/json'}
url = utils.build_url(self.url + "comment/v1/comment/list/", {'market_hash': market_hash})
logger.debug(url)
resp = yield treq.get(url, headers=header)
logger.debug(resp)
comment_info = yield treq.json_content(resp)
logger.debug('upload file info to market confirm: %s', comment_info)
return comment_info['data']
def got_response(response):
if response.code in success_codes:
action.addSuccessFields(response_code=response.code)
d = json_content(response)
d.addCallback(lambda decoded_body:
(decoded_body, response.headers))
return d
else:
d = content(response)
d.addCallback(error, response.code)
return d
def query_carousel(self):
logger.debug('status: in query carousel')
url = self.url + 'main/v1/carousel/list/'
logger.debug("query carousel url: %s", url)
header = {'Content-Type': 'application/json', 'MARKET-KEY': self.public_key,
'MARKET-TOKEN': self.token}
resp = yield treq.get(url=url, headers=header)
confirm_info = yield treq.json_content(resp)
print(confirm_info)
logger.debug("carousel response: %s", confirm_info)
return confirm_info['data']
def query_data(self, market_hash):
url = self.url + 'user_data/v1/uploaded_file/item/'
header = {
"MARKET-KEY": self.public_key,
"MARKET-TOKEN": self.token,
'Content-Type': 'application/json'
}
url = utils.build_url(url, {'market_hash': market_hash})
resp = yield treq.get(url, headers=header, persistent=False)
data_info = yield treq.json_content(resp)
return data_info
def add_comment_by_hash(self, market_hash, comment=""):
comment_address = ECCipher.get_address_from_public_key(self.account.public_key)
logger.debug(comment_address)
header = {"MARKET-KEY": self.public_key, "MARKET-TOKEN": self.token, 'Content-Type': 'application/json'}
data = {'public_key': self.public_key, 'market_hash': market_hash, 'content': comment}
url = utils.build_url(self.url + "comment/v1/comment/add/", {'market_hash': market_hash})
resp = yield treq.post(url, headers=header, json=data, persistent=False)
comment_info = yield treq.json_content(resp)
logger.debug('upload file info to market confirm: %s', comment_info)
return comment_info['status']
def test_get_server_image(self):
"""
Test to verify :func:`get_image` on ``GET /v2.0/<tenant_id>/images/<image_id>``
"""
get_server_image = request(self, self.root, "GET", self.uri + '/images/test-image-id')
get_server_image_response = self.successResultOf(get_server_image)
get_server_image_response_body = self.successResultOf(
treq.json_content(get_server_image_response))
self.assertEqual(get_server_image_response.code, 200)
self.assertEqual(get_server_image_response_body['image']['id'], 'test-image-id')
self.assertEqual(get_server_image_response_body['image']['status'], 'ACTIVE')
def test_get_server_ips(self):
"""
Test to verify :func:`get_ips` on ``GET /v2.0/<tenant_id>/servers/<server_id>/ips``
"""
get_server_ips = request(
self, self.root, "GET",
self.uri + '/servers/' + self.server_id + '/ips')
get_server_ips_response = self.successResultOf(get_server_ips)
get_server_ips_response_body = self.successResultOf(
treq.json_content(get_server_ips_response))
self.assertEqual(get_server_ips_response.code, 200)
list_servers_detail = request(self, self.root, "GET",
self.uri + '/servers/detail')
list_servers_detail_response = self.successResultOf(
list_servers_detail)
list_servers_detail_response_body = self.successResultOf(
treq.json_content(list_servers_detail_response))
self.assertEqual(
get_server_ips_response_body['addresses'],
list_servers_detail_response_body['servers'][0]['addresses'])
def query_records(self, address):
url = self.url + 'records/v1/record/'
header = {
"MARKET-KEY": self.public_key,
"MARKET-TOKEN": self.token,
'Content-Type': 'application/json'
}
url = utils.build_url(url, {'address': address})
resp = yield treq.get(url, headers=header, persistent=False)
data_info = yield treq.json_content(resp)
return data_info