async def mistake_url(self, message, url):
client = AsyncHTTPClient()
try:
tornado_future = client.fetch(url)
future = to_asyncio_future(tornado_future)
response = await future
with open('/tmp/plop.png', 'wb') as f:
f.write(response.body)
except:
return
tornado_future = client.fetch(Images.MISTAKE_URL)
future = to_asyncio_future(tornado_future)
response = await future
with open('/tmp/mistake.png', 'wb') as f:
f.write(response.body)
img = Image.open('/tmp/plop.png')
mistake = Image.open('/tmp/mistake.png')
img = img.resize((505, 557))
mistake.paste(img, (mistake.width - 505, mistake.height - 557))
mistake.save('/tmp/lol.png')
await self.send_file(
message.channel,
'/tmp/lol.png',
delete_after = 30,
)
async def mistake_user(self, message, user_id):
member = message.server.get_member(str(user_id))
if member is None:
return
client = AsyncHTTPClient()
tornado_future = client.fetch(member.avatar_url)
future = to_asyncio_future(tornado_future)
response = await future
with open('/tmp/plop.png', 'wb') as f:
f.write(response.body)
tornado_future = client.fetch(Images.MISTAKE_URL)
future = to_asyncio_future(tornado_future)
response = await future
with open('/tmp/mistake.png', 'wb') as f:
f.write(response.body)
avatar = Image.open('/tmp/plop.png')
mistake = Image.open('/tmp/mistake.png')
avatar = avatar.resize((505, 557))
mistake.paste(avatar, (mistake.width - 505, mistake.height - 557))
mistake.save('/tmp/lol.png')
await self.send_file(
message.channel,
'/tmp/lol.png',
delete_after = 30,
)
async def _call_thrice(call):
# gen_test setup somehow interferes with coroutines and futures
# code tested manually works without such "decorations" but for testing this workaround was the bes I've found
if memoize_configuration.force_asyncio:
res2 = await asyncio.gather(call(), call(), call())
else:
res2 = await asyncio.gather(to_asyncio_future(call()), to_asyncio_future(call()), to_asyncio_future(call()))
return res2
def go():
f = submit("1 + 1")
f = to_asyncio_future(f)
resp = yield trollius.From(f)
while True:
msg = to_asyncio_future(resp.read())
msg = yield trollius.From(msg)
if msg is None:
break
self.assertEqual(msg.status_code, 200)
self.assertEqual(msg.data[0], 2)
def commit(self):
"""
.. todo:: Use bulk insert and bulk delete.
"""
for resource in self._added_resources:
yield from to_asyncio_future(resource.save())
for resource in self._saved_resources.values():
yield from to_asyncio_future(resource.save())
for resource in self._deleted_resources.values():
yield from to_asyncio_future(resource.delete())
return None
def commit(self):
"""
.. todo:: Use bulk insert and bulk delete.
"""
for resource in self._added_resources:
yield from to_asyncio_future(resource.save())
for resource in self._saved_resources.values():
yield from to_asyncio_future(resource.save())
for resource in self._deleted_resources.values():
yield from to_asyncio_future(resource.delete())
return None
def process_config(self):
config = super().process_config()
if 'ETHEREUM_NODE_URL' in os.environ:
config['ethereum'] = {'url': os.environ['ETHEREUM_NODE_URL']}
if 'DEFAULT_GASPRICE' in os.environ:
if 'ethereum' not in config:
config['ethereum'] = {}
config['ethereum']['default_gasprice'] = os.environ[
'DEFAULT_GASPRICE']
if 'ethereum' in config:
if 'ETHEREUM_NETWORK_ID' in os.environ:
config['ethereum']['network_id'] = os.environ[
'ETHEREUM_NETWORK_ID']
else:
config['ethereum'][
'network_id'] = self.asyncio_loop.run_until_complete(
to_asyncio_future(
JsonRPCClient(
config['ethereum']['url']).net_version()))
configure_logger(services_log)
return config
async def fetch(url):
client = AsyncHTTPClient()
tornado_future = client.fetch(url)
future = to_asyncio_future(tornado_future)
response = await future
return response.body
async def test_transaction_nonce_lock(self):
"""Spams transactions with the same nonce, and ensures the server rejects all but one"""
no_tests = 20
txs = []
tx = await self.get_tx_skel(FAUCET_PRIVATE_KEY, TEST_ADDRESS, 10**10)
dtx = decode_transaction(tx)
txs.append(sign_transaction(tx, FAUCET_PRIVATE_KEY))
for i in range(11, 10 + no_tests):
tx = await self.get_tx_skel(FAUCET_PRIVATE_KEY, TEST_ADDRESS,
10**i)
self.assertEqual(decode_transaction(tx).nonce, dtx.nonce)
txs.append(sign_transaction(tx, FAUCET_PRIVATE_KEY))
responses = await asyncio.gather(*(to_asyncio_future(
self.fetch("/tx", method="POST", body={"tx": tx})) for tx in txs))
ok = 0
bad = 0
for resp in responses:
if resp.code == 200:
ok += 1
else:
bad += 1
self.assertEqual(ok, 1)
self.assertEqual(bad, no_tests - 1)
# TODO: deal with lingering ioloop tasks better
await asyncio.sleep(1)
async def fetch(self, prefix, path, params):
url = url_concat(prefix + path, params)
print(url)
tornado_future = self.client.fetch(url)
future = to_asyncio_future(tornado_future)
response = await future
return json_decode(response.body)
def get(self, identifier, required=False):
"""
"""
typename, resource_id = identifier
resource_class = self.api.get_resource_class(typename)
resource = yield from to_asyncio_future(resource_class.objects.get(resource_id))
if required and resource is None:
raise jsonapi.base.errors.ResourceNotFound(identifier)
return resource
def main():
AsyncIOMainLoop().install()
loop = asyncio.get_event_loop()
afuture = async_timeout()
tfuture = to_asyncio_future(tornado_timeout())
tasks = asyncio.wait([afuture, tfuture])
loop.run_until_complete(tasks)
loop.close()
async def get_json(self, endpoint, params={}):
url = url_concat(TwitchAPI.BASE_URL + endpoint, params)
request = HTTPRequest(
url = url,
headers = self.api_headers
)
tornado_future = self.client.fetch(request)
future = to_asyncio_future(tornado_future)
response = await future
return json_decode(response.body)
def get(self, identifier, required=False):
"""
"""
typename, resource_id = identifier
resource_class = self.api.get_resource_class(typename)
resource = yield from to_asyncio_future(
resource_class.objects.get(resource_id))
if required and resource is None:
raise jsonapi.base.errors.ResourceNotFound(identifier)
return resource
async def update_token_cache(self, contract_address, *eth_addresses):
if len(eth_addresses) == 0:
return
async with self.db:
if contract_address == "*":
tokens = await self.db.fetch(
"SELECT contract_address FROM tokens")
else:
tokens = [{'contract_address': contract_address}]
futures = []
for token in tokens:
for address in eth_addresses:
# data for `balanceOf(address)`
data = "0x70a08231000000000000000000000000" + address[2:]
f = to_asyncio_future(
self.eth.eth_call(to_address=token['contract_address'],
data=data))
futures.append((token['contract_address'], address, f))
# wait for all the jsonrpc calls to finish
await asyncio.gather(*[f[2] for f in futures], return_exceptions=True)
bulk_update = []
bulk_delete = []
for contract_address, eth_address, f in futures:
try:
value = f.result()
# value of "0x" means something failed with the contract call
if value == "0x0000000000000000000000000000000000000000000000000000000000000000" or value == "0x":
if value == "0x":
log.warning(
"calling balanceOf for contract {} failed".format(
contract_address))
bulk_delete.append((contract_address, eth_address))
else:
value = hex(
parse_int(value)) # remove hex padding of value
bulk_update.append((contract_address, eth_address, value))
except:
log.exception(
"WARNING: failed to update token cache of '{}' for address: {}"
.format(contract_address, eth_address))
continue
async with self.db:
await self.db.executemany(
"INSERT INTO token_balances (contract_address, eth_address, value) VALUES ($1, $2, $3) "
"ON CONFLICT (contract_address, eth_address) DO UPDATE set value = EXCLUDED.value",
bulk_update)
await self.db.executemany(
"DELETE FROM token_balances WHERE contract_address = $1 AND eth_address = $2",
bulk_delete)
await self.db.commit()
async def get_json(self, endpoint, params={}, token=None):
url = url_concat(TwitchAPI.BASE_URL + endpoint, params)
request = HTTPRequest(
url = url,
headers = self.api_headers(token)
)
tornado_future = self.client.fetch(request)
future = to_asyncio_future(tornado_future)
response = await future
return json_decode(response.body)
def fetch(self, url: str, encoding: str = "utf-8") -> HTTPResponse:
"""Fetch url and return ``tornado.httpclient.HTTPResponse`` object.
Response body is decoded by ``encoding`` and set ``text`` property of
the response. If failed to decode, ``text`` property will be ``None``.
"""
response = yield from to_asyncio_future(AsyncHTTPClient().fetch(url, raise_error=False))
try:
response.text = response.body.decode(encoding)
except UnicodeDecodeError:
response.text = None
return response
async def test_send_large_file(self):
"""Tests uploading a large avatar and makes sure it doesn't block
other processes"""
capitalised = 'BobSmith'
async with self.pool.acquire() as con:
await con.execute(
"INSERT INTO users (username, toshi_id) VALUES ($1, $2)",
capitalised, TEST_ADDRESS)
# saves a generated file in the /tmp dir to speed up
# multiple runs of the same test
tmpfile = "/tmp/toshi-testing-large-file-test-2390.jpg"
if os.path.exists(tmpfile):
jpeg = open(tmpfile, 'rb').read()
else:
# generates a file just under 100mb (100mb being the max size upload supported)
jpeg = blockies.create(TEST_ADDRESS_2,
size=2390,
scale=12,
format='JPEG')
with open(tmpfile, 'wb') as f:
f.write(jpeg)
print("size: {} MB".format(len(jpeg) / 1024 / 1024))
boundary = uuid4().hex
headers = {
'Content-Type': 'multipart/form-data; boundary={}'.format(boundary)
}
files = [('avatar.jpg', jpeg)]
body = body_producer(boundary, files)
f1 = self.fetch_signed("/user",
signing_key=TEST_PRIVATE_KEY,
method="PUT",
body=body,
headers=headers)
# make sure the avatar upload has begun
await asyncio.sleep(1)
f2 = self.fetch("/v1/user/{}".format(TEST_ADDRESS), method="GET")
def f1done(r):
assert f2.done()
def f2done(r):
assert not f1.done()
loop = IOLoop.current()
loop.add_future(f1, f1done)
loop.add_future(f2, f2done)
await asyncio.wait([to_asyncio_future(f) for f in [f1, f2]], timeout=5)
async def kill_user(self, message, user_id):
member = message.server.get_member(str(user_id))
if member is None:
return
client = AsyncHTTPClient()
tornado_future = client.fetch(member.avatar_url)
future = to_asyncio_future(tornado_future)
response = await future
with open('/tmp/plop.png', 'wb') as f:
f.write(response.body)
tornado_future = client.fetch(Images.GUN_URL)
future = to_asyncio_future(tornado_future)
response = await future
with open('/tmp/gun.svg', 'wb') as f:
f.write(response.body)
avatar = Image.open('/tmp/plop.png')
gun = Image.open('/tmp/gun.svg')
gun.resize(avatar.size)
total = Image.new('RGB', (avatar.width * 2, avatar.height))
total.paste(avatar, (0, 0))
total.paste(gun, (avatar.width, 0))
total.save('/tmp/lol.png')
await self.send_file(
message.channel,
'/tmp/lol.png',
delete_after = 30,
content = '{} is now dead 👍'.format(member.mention)
)
def query_size(self,
typename,
*,
order=None,
limit=None,
offset=None,
filters=None):
"""
"""
query = self._build_query(typename,
order=order,
limit=limit,
offset=offset,
filters=filters)
return to_asyncio_future(query.count())
async def update_cloudflare_ips():
global CLOUDFLARE_IPS
urls = ['https://www.cloudflare.com/ips-v4',
'https://www.cloudflare.com/ips-v6']
client = AsyncHTTPClient()
coros = [to_asyncio_future(client.fetch(url)) for url in urls]
rs, _ = await asyncio.wait(coros)
new = []
for r in rs:
r = r.result()
new.extend(ipaddress.ip_network(line)
for line in r.body.decode('utf-8').splitlines())
CLOUDFLARE_IPS = new
async def fetch_definitions(self, term):
url = url_concat(UrbanDictionary.DEFINE_URL, dict(term=term))
request = HTTPRequest(
url = url,
headers = {
'X-Mashape-Key': self.api_key,
'Accept' : 'text/plain'
}
)
tornado_future = self.client.fetch(request)
future = to_asyncio_future(tornado_future)
response = await future
data = json_decode(response.body)
return data['list']
async def _read(self, *, timeout=None):
f = self.con.read_message()
if timeout:
try:
result = await asyncio.wait_for(to_asyncio_future(f), timeout)
except asyncio.TimeoutError as e:
# reset the connection's read state
self.con.read_future = None
return None
else:
result = await f
if result:
result = tornado.escape.json_decode(result)
return result
async def update_cloudflare_ips():
global CLOUDFLARE_IPS
urls = [
'https://www.cloudflare.com/ips-v4',
'https://www.cloudflare.com/ips-v6'
]
client = AsyncHTTPClient()
coros = [to_asyncio_future(client.fetch(url)) for url in urls]
rs, _ = await asyncio.wait(coros)
new = []
for r in rs:
r = r.result()
new.extend(
ipaddress.ip_network(line)
for line in r.body.decode('utf-8').splitlines())
CLOUDFLARE_IPS = new
def ws_connect(self, url: str, timeout: float = None) -> WebSocketClientConnection:
"""Make WebSocket connection to the url.
Retries up to _max (default: 20) times. Client connections made by this
method are closed after each test method.
"""
st = time.perf_counter()
timeout = timeout or self.timeout
while (time.perf_counter() - st) < timeout:
try:
ws = yield from to_asyncio_future(websocket_connect(url))
except ConnectionRefusedError:
yield from self.wait()
continue
else:
self._ws_connections.append(ws)
return ws
raise ConnectionRefusedError("WebSocket connection refused: {}".format(url))
def maybe_future(obj):
"""Return an asyncio Future
Use instead of gen.maybe_future
For our compatibility, this must accept:
- asyncio coroutine (gen.maybe_future doesn't work in tornado < 5)
- tornado coroutine (asyncio.ensure_future doesn't work)
- scalar (asyncio.ensure_future doesn't work)
- concurrent.futures.Future (asyncio.ensure_future doesn't work)
- tornado Future (works both ways)
- asyncio Future (works both ways)
"""
if inspect.isawaitable(obj):
# already awaitable, use ensure_future
return asyncio.ensure_future(obj)
elif isinstance(obj, concurrent.futures.Future):
return asyncio.wrap_future(obj)
else:
return to_asyncio_future(gen.maybe_future(obj))
def maybe_future(obj):
"""Return an asyncio Future
Use instead of gen.maybe_future
For our compatibility, this must accept:
- asyncio coroutine (gen.maybe_future doesn't work in tornado < 5)
- tornado coroutine (asyncio.ensure_future doesn't work)
- scalar (asyncio.ensure_future doesn't work)
- concurrent.futures.Future (asyncio.ensure_future doesn't work)
- tornado Future (works both ways)
- asyncio Future (works both ways)
"""
if inspect.isawaitable(obj):
# already awaitable, use ensure_future
return asyncio.ensure_future(obj)
elif isinstance(obj, concurrent.futures.Future):
return asyncio.wrap_future(obj)
else:
return to_asyncio_future(gen.maybe_future(obj))
def process_config(self):
config = super().process_config()
if 'ETHEREUM_NODE_URL' in os.environ:
config['ethereum'] = {'url': os.environ['ETHEREUM_NODE_URL']}
if 'MONITOR_ETHEREUM_NODE_URL' in os.environ:
config['monitor'] = {
'url': os.environ['MONITOR_ETHEREUM_NODE_URL']
}
if 'ethereum' in config:
if 'ETHEREUM_NETWORK_ID' in os.environ:
config['ethereum']['network_id'] = os.environ[
'ETHEREUM_NETWORK_ID']
else:
config['ethereum'][
'network_id'] = self.asyncio_loop.run_until_complete(
to_asyncio_future(
JsonRPCClient(
config['ethereum']['url']).net_version()))
return config
async def access_token(self, code, state):
client = AsyncHTTPClient()
payload = (
('client_id', self.client_id),
('client_secret', self.client_secret),
('grant_type', 'authorization_code'),
('redirect_uri', Twitch.REDIRECT_URI),
('code', code),
('state', state),
)
url = Twitch.TOKEN_URL
request = HTTPRequest(
url = url,
method = 'POST',
body = urlencode(payload)
)
tornado_future = client.fetch(request)
future = to_asyncio_future(tornado_future)
response = await future
data = json_decode(response.body)
return data['access_token']
async def access_token(self, code, state):
client = AsyncHTTPClient()
payload = (
('client_id', self.client_id),
('client_secret', self.client_secret),
('grant_type', 'authorization_code'),
('redirect_uri', Twitch.REDIRECT_URI),
('code', code),
('state', state),
)
url = Twitch.TOKEN_URL
request = HTTPRequest(
url = url,
method = 'POST',
body = urlencode(payload)
)
tornado_future = client.fetch(request)
future = to_asyncio_future(tornado_future)
response = await future
data = json_decode(response.body)
return data['access_token']
def convert(*args, **kwargs):
if default_kwargs:
kwargs = merge(default_kwargs, kwargs)
return to_asyncio_future(fn(*args, **kwargs))
def __await__(self):
return to_asyncio_future(self._started).__await__()
def convert(*args, **kwargs):
return to_asyncio_future(fn(*args, **kwargs))
__author__ = 'zhangxa'
from tornado import gen
from tornado.platform.asyncio import to_asyncio_future, AsyncIOMainLoop
from tornado.httpclient import AsyncHTTPClient
import asyncio
@gen.coroutine
def tornado_coroutine():
cli = AsyncHTTPClient()
response = yield cli.fetch("http://www.baidu.com")
print(response.body)
AsyncIOMainLoop().install()
asyncio.get_event_loop().run_until_complete(
to_asyncio_future(tornado_coroutine()))
def handle(msg):
yield from to_asyncio_future(
dispatch.Dispatch(telegram_send_message, telegram_users).run(msg))
def query_size(self, typename, *, order=None, limit=None, offset=None, filters=None):
"""
"""
query = self._build_query(typename, order=order, limit=limit, offset=offset, filters=filters)
return to_asyncio_future(query.count())
def __await__(self):
return to_asyncio_future(self._started).__await__()
def get_greetings():
http_client = AsyncHTTPClient()
response = yield from to_asyncio_future(http_client.fetch(URL))
return response.body.decode('utf-8')
def handle(msg):
yield from to_asyncio_future(
dispatch.Dispatch(telegram_send_message, telegram_users).run(msg)
)
def convert(*args, **kwargs):
if default_kwargs:
kwargs = merge(default_kwargs, kwargs)
return to_asyncio_future(fn(*args, **kwargs))
__author__ = 'zhangxa'
from tornado import gen
from tornado.platform.asyncio import to_asyncio_future,AsyncIOMainLoop
from tornado.httpclient import AsyncHTTPClient
import asyncio
@gen.coroutine
def tornado_coroutine():
cli = AsyncHTTPClient()
response = yield cli.fetch("http://www.baidu.com")
print(response.body)
AsyncIOMainLoop().install()
asyncio.get_event_loop().run_until_complete(to_asyncio_future(tornado_coroutine()))
