def test_fetch_expired_credentials(self):
cfg_client = httpclient.AsyncHTTPClient()
role = uuid.uuid4().hex
access_key = uuid.uuid4().hex
secret_key = uuid.uuid4().hex
token = uuid.uuid4().hex
url = self.get_url(
'/latest/meta-data/iam/security-credentials/{}?role={}&access_'
'key={}&secret_key={}&token={}'.format(
role, role, access_key, secret_key, token))
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = '/tmp/{}'.format(
access_key)
with mock.patch('tornado_aws.config.INSTANCE_ENDPOINT', url):
cfg = config.Authorization('default', client=cfg_client)
with self.client_with_default_creds(
's3', endpoint=self.get_url('/api')) as obj:
obj._auth_config = cfg
with mock.patch.object(obj._client, 'fetch') as fetch:
future1 = concurrent.Future()
future1.set_exception(self.mock_auth_exception())
future2 = concurrent.Future()
future2.set_result(self.mock_ok_response())
fetch.side_effect = [future1, future2]
result = yield obj.fetch('GET', '/api')
self.assertEqual(result.code, 200)
def test_fetch_credentials_and_request_fail(self):
cfg_client = httpclient.AsyncHTTPClient()
role = uuid.uuid4().hex
access_key = uuid.uuid4().hex
secret_key = uuid.uuid4().hex
token = uuid.uuid4().hex
url = self.get_url(
'/latest/meta-data/iam/security-credentials/{}?role={}&access_'
'key={}&secret_key={}&token={}'.format(
role, role, access_key, secret_key, token))
with mock.patch('tornado_aws.config.INSTANCE_ENDPOINT', url):
cfg = config.Authorization('default', client=cfg_client)
with mock.patch.object(cfg, '_get_role_async') as get_role:
role_future = concurrent.Future()
role_future.set_result(role)
get_role.return_value = role_future
with self.client_with_default_creds(
's3', endpoint=self.get_url('/api')) as obj:
obj._auth_config = cfg
with mock.patch.object(obj._client, 'fetch') as fetch:
future1 = concurrent.Future()
future1.set_exception(self.mock_auth_exception())
future2 = concurrent.Future()
future2.set_exception(self.mock_error_exception())
fetch.side_effect = [future1, future2]
with self.assertRaises(exceptions.AWSError):
yield obj.fetch('GET', '/api')
def setup(self):
"""Setup two scopes and two "locks"."""
self.scope_a = rw.scope.Scope()
self.scope_a['name'] = 'a'
self.lock_a = concurrent.Future()
self.scope_b = rw.scope.Scope()
self.scope_b['name'] = 'b'
self.lock_b = concurrent.Future()
@rw.scope.inject
def get_name(name):
return name
@tornado.gen.coroutine
def thread_a():
yield self.lock_a
raise tornado.gen.Return(get_name())
@tornado.gen.coroutine
def thread_b():
yield self.lock_b
raise tornado.gen.Return(get_name())
with self.scope_a():
future_a = thread_a()
with self.scope_b():
future_b = thread_b()
return future_a, future_b
def test_observer(user,
object_service,
object_name,
data,
access_type,
columns=None,
*args,
**kwargs):
assert object_name == "test"
assert access_type == AccessType.READ or access_type == AccessType.INSERT or access_type == AccessType.UPDATE or access_type == AccessType.DELETE
logging.info(
"Test object observer is called with [{}, {}, {}, {}, {}, {}].".format(
user, object_service, object_name, data, access_type, columns))
if AccessType.INSERT == access_type:
val = concurrent.Future()
val.set_result(data)
return (val)
if AccessType.UPDATE == access_type or AccessType.DELETE == access_type:
r = ({"error": None, "n": 1, "ok": 1, "updatedExisting": 1})
val = concurrent.Future()
val.set_result({"_id": r})
return (val)
find_one = kwargs.get("find_one", False)
if find_one:
val = concurrent.Future()
val.set_result({"_id": data})
return (val)
else:
Result = namedtuple("CosmosEmptyResultSet", "fetch_next")
val = concurrent.Future()
val.set_result(False)
return (Result(fetch_next=val))
def test_nested_groutine_with_double_gyield_2():
'''Ensure nested groutine + 2 or more yield works'''
future1 = concurrent.Future()
future2 = concurrent.Future()
def callback1():
future1.set_result(1234)
def callback2():
future2.set_result(4321)
@greenado.groutine
@greenado.generator
def nested_groutine():
IOLoop.current().add_callback(callback1)
result = (yield future1) + 1
IOLoop.current().add_callback(callback2)
result += (yield future2) + 1
raise gen.Return(result)
@greenado.groutine
@greenado.generator
def _main():
retval = (yield nested_groutine()) + 1
raise gen.Return(retval)
main_retval = IOLoop.current().run_sync(_main)
assert main_retval == 5558
def test_nested_groutine_with_double_gyield_1():
'''Ensure nested groutine + 2 or more gyield works'''
future1 = concurrent.Future()
future2 = concurrent.Future()
def callback1():
future1.set_result(1234)
def callback2():
future2.set_result(4321)
@greenado.groutine
def nested_groutine():
IOLoop.current().add_callback(callback1)
result = greenado.gyield(future1) + 1
IOLoop.current().add_callback(callback2)
result += greenado.gyield(future2) + 1
return result
@greenado.groutine
def _main():
return greenado.gyield(nested_groutine()) + 1
main_retval = IOLoop.current().run_sync(_main)
assert main_retval == 5558
def test_on_channel_open_reset_confirmation_bookkeeping(self):
self.client.message_number = 2
self.client.messages = {1: concurrent.Future(), 2: concurrent.Future()}
self.client.state = amqp.Client.STATE_CONNECTING
self.assertTrue(self.client.connecting)
self.client.on_channel_open(self.client.channel)
self.assertEqual(self.client.message_number, 0)
self.assertEqual(self.client.messages, {})
self.assertEqual(self.client.channel.confirm_delivery.call_count, 1)
def setUp(self):
super(AsyncHTTPTestCase, self).setUp()
self.correlation_id = str(uuid.uuid4())
self.exchange = str(uuid.uuid4())
self.get_delivered_message = concurrent.Future()
self.get_returned_message = concurrent.Future()
self.queue = str(uuid.uuid4())
self.routing_key = str(uuid.uuid4())
self.ready = locks.Event()
amqp.install(self._app, self.io_loop, **{
'on_ready_callback': self.on_amqp_ready,
'enable_confirmations': self.CONFIRMATIONS,
'on_return_callback': self.on_message_returned,
'url': 'amqp://*****:*****@127.0.0.1:5672/%2f'})
self.io_loop.start()
def refresh(self):
"""Load dynamic credentials from the AWS Instance Metadata and user
data HTTP API.
:raises: tornado_aws.exceptions.NoCredentialsError
"""
LOGGER.debug('Refreshing EC2 IAM Credentials')
future = concurrent.Future() if self._is_async else None
try:
result = self._fetch_credentials()
if concurrent.is_future(result):
def on_complete(response):
exception = response.exception()
if exception:
if isinstance(exception, httpclient.HTTPError) and \
exception.code == 599:
future.set_exception(
exceptions.NoCredentialsError())
else:
future.set_exception(exception)
return
self._assign_credentials(response.result())
future.set_result(True)
self._ioloop.add_future(result, on_complete)
else:
self._assign_credentials(result)
except (httpclient.HTTPError, OSError) as error:
LOGGER.error('Error Fetching Credentials: %s', error)
raise exceptions.NoCredentialsError
return future
def _execute(self, parts, expectation=None, format_callback=None):
future = concurrent.Future()
if isinstance(self._execute_result, Exception):
future.set_exception(self._execute_result)
else:
future.set_result(self._execute_result)
return future
def _execute(self, action, parameters, attempt, measurements):
"""Invoke a DynamoDB action
:param str action: DynamoDB action to invoke
:param dict parameters: parameters to send into the action
:param int attempt: Which attempt number this is
:param list measurements: A list for accumulating request measurements
:rtype: tornado.concurrent.Future
"""
future = concurrent.Future()
start = time.time()
def handle_response(request):
"""Invoked by the IOLoop when fetch has a response to process.
:param tornado.concurrent.Future request: The request future
"""
self._on_response(action, parameters.get('TableName', 'Unknown'),
attempt, start, request, future, measurements)
ioloop.IOLoop.current().add_future(
self._client.fetch('POST',
'/',
body=json.dumps(parameters).encode('utf-8'),
headers={
'x-amz-target':
'DynamoDB_20120810.{}'.format(action),
'Content-Type':
'application/x-amz-json-1.0',
}), handle_response)
return future
def __init__(self, server_state, context):
super(Http2Layer, self).__init__(server_state, context)
self.src_conn = Connection(
self.src_stream, client_side=False,
conn_type="source",
on_request=self.on_request,
on_settings=self.on_src_settings,
on_window_updates=self.on_src_window_updates,
on_priority_updates=self.on_src_priority_updates,
on_reset=self.on_src_reset,
on_terminate=self.on_src_terminate,
readonly=(context.mode == "replay"))
self.dest_conn = Connection(
self.dest_stream, client_side=True,
conn_type="destination",
on_response=self.on_response,
on_push=self.on_push,
on_settings=self.on_dest_settings,
on_window_updates=self.on_dest_window_updates,
on_terminate=self.on_dest_terminate,
on_reset=self.on_dest_reset)
self.streams = dict()
self.src_to_dest_ids = dict([(0, 0)])
self.dest_to_src_ids = dict([(0, 0)])
self._future = concurrent.Future()
def create_table_expecting_raise(self, exception, future_exception=None):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
future = concurrent.Future()
future.set_exception(future_exception or exception)
fetch.return_value = future
with self.assertRaises(exception):
yield self.client.create_table(self.generic_table_definition())
def on_replication_info(_):
common.maybe_raise_exception(failover_future)
LOGGER.debug('Failover closing current read-only connection')
self._closing = True
database = self._connection.database
self._connection.close()
self._connected.clear()
self._connect_future = concurrent.Future()
info = failover_future.result()
LOGGER.debug('Failover connecting to %s:%s', info['master_host'],
info['master_port'])
self._connection = _Connection(info['master_host'],
info['master_port'], database,
self._read, self._on_closed,
self.io_loop, self._clustering)
# When the connection is re-established, re-run the command
self.io_loop.add_future(
self._connect_future, lambda f: self._connection.execute(
command._replace(connection=self._connection), future))
# Use the normal connection processing flow when connecting
self.io_loop.add_future(self._connection.connect(),
self._on_connected)
def test_consuming_not_coroutine(self):
client = yield self.create_connection()
queue_name = self.get_random_name("tc2")
routing_key = self.get_random_name()
channel = yield client.channel()
exchange = yield channel.declare_exchange('direct', auto_delete=True)
queue = yield channel.declare_queue(queue_name, auto_delete=True)
yield queue.bind(exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
f = concurrent.Future()
def handle(message):
message.ack()
self.assertEqual(message.body, body)
self.assertEqual(message.routing_key, routing_key)
f.set_result(True)
yield queue.consume(handle)
yield exchange.publish(
Message(body, content_type='text/plain', headers={'foo': 'bar'}),
routing_key)
if not f.done():
yield f
yield queue.unbind(exchange, routing_key)
yield exchange.delete()
yield wait((client.close(), client.closing))
def on_connected(cf):
"""Invoked by the future returned by self._connect"""
if cf.exception():
future.set_exception(cf.exception())
return
# Get the psycopg2 connection object and cursor
conn = cf.result()
cursor = self._get_cursor(conn)
def completed(qf):
"""Invoked by the IOLoop when the future has completed"""
if qf.exception():
self._incr_exceptions(conn)
err = qf.exception()
LOGGER.debug('Cleaning cursor due to exception: %r', err)
self._exec_cleanup(cursor, conn.fileno())
future.set_exception(err)
else:
self._incr_executions(conn)
value = Results(cursor, self._exec_cleanup, conn.fileno())
future.set_result(value)
# Setup a callback to wait on the query result
self._futures[conn.fileno()] = concurrent.Future()
# Add the future to the IOLoop
self._ioloop.add_future(self._futures[conn.fileno()], completed)
# Get the cursor, execute the query
func = getattr(cursor, method)
try:
func(query, parameters)
except Exception as error:
future.set_exception(error)
def start(self, seed_partitions):
"""
Connects to zookeeper and collects member and partition data.
Leverages the `create_attempt()` and ``wait_on_event()`` helper
functions in order to bridge the gap between threaded async
and tornado async.
Returns a ``Future`` instance once done so that coroutine
methods may yield to it.
"""
log.info("Starting partitioner for group '%s'", self.group_name)
f = concurrent.Future()
attempt = create_attempter(f)
attempt(self.connect)
wait_on_event(self.connected)
attempt(self.party.start)
attempt(self.shared_set.start)
attempt(self.party.join)
attempt(self.add_partitions, seed_partitions)
if f.done():
return f
wait_on_event(self.members_collected)
wait_on_event(self.partitions_collected)
f.set_result(None)
return f
def acquire(self):
blocker = self._waiters[-1] if self.acquired() else None
future = concurrent.Future()
self._waiters.append(future)
if blocker:
yield blocker
raise gen.Return(self._lock_context())
def wait_for_change(key, watcher):
future = concurrent.Future()
callback = lambda *args: future.set_result(args)
watcher.add_listener(key, callback)
result = yield future
watcher.remove_listener(key, callback)
raise gen.Return(result)
def _publisher_confirmation_future(self, name, exchange, routing_key,
properties):
"""Return a future a publisher confirmation result that enables
consumers to block on the confirmation of a published message.
Two internal dicts are used for keeping track of state.
Consumer._delivery_tags is a dict of connection names that keeps
the last delivery tag expectation and is used to correlate the future
with the delivery tag that is expected to be confirmed from RabbitMQ.
This for internal use and should not be extended or used directly.
:param str name: The connection name for the future
:param str exchange: The exchange the message was published to
:param str routing_key: The routing key that was used
:param properties: The AMQP message properties for the delivery
:type properties: pika.spec.Basic.Properties
:rtype: concurrent.Future.
"""
if self._connections[name].publisher_confirmations:
future = concurrent.Future()
self._connections[name].add_confirmation_future(
exchange, routing_key, properties, future)
return future
def get_app(self):
# In test cases including this mixin a WebSocket server is created.
# The server creates a new client on each request. This client should
# forward messages to a WebSocket echo server. In order to test the
# communication, some of the tests create another client that connects
# to the server, e.g.:
# ws-client -> ws-server -> ws-forwarding-client -> ws-echo-server
# Messages arriving to the echo server are returned back to the client:
# ws-echo-server -> ws-forwarding-client -> ws-server -> ws-client
self.apiurl = self.get_wss_url('/echo')
self.api_close_future = concurrent.Future()
self.deployer = base.Deployer(
self.apiurl, manage.DEFAULT_API_VERSION, io_loop=self.io_loop)
self.tokens = auth.AuthenticationTokenHandler(io_loop=self.io_loop)
echo_options = {
'close_future': self.api_close_future,
'io_loop': self.io_loop,
}
ws_options = {
'apiurl': self.apiurl,
'auth_backend': self.auth_backend,
'deployer': self.deployer,
'io_loop': self.io_loop,
'tokens': self.tokens,
}
return web.Application([
(r'/echo', helpers.EchoWebSocketHandler, echo_options),
(r'/ws', handlers.WebSocketHandler, ws_options),
])
def send(self, request, xid=None):
f = concurrent.Future()
if self.closing:
f.set_exception(exc.ConnectError(self.host, self.port))
return f
if request.special_xid:
xid = request.special_xid
payload_log.debug("[SEND] (xid: %s) %s", xid, request)
payload = request.serialize(xid)
payload = size_struct.pack(len(payload)) + payload
self.opcode_xref[xid] = request.opcode
if xid in protocol.SPECIAL_XIDS:
self.pending_specials[xid].append(f)
else:
self.pending[xid] = f
def handle_write(write_future):
try:
write_future.result()
except Exception:
self.abort()
try:
self.stream.write(payload).add_done_callback(handle_write)
except Exception:
self.abort()
return f
def mock_websocket_connect(self):
"""Mock the guiserver.clients.websocket_connect function."""
future = concurrent.Future()
future.set_result(mock.Mock())
mock_websocket_connect = mock.Mock(return_value=future)
return mock.patch(
'guiserver.handlers.websocket_connect', mock_websocket_connect)
def send(self, message):
"""
Sends a serialized request to the broker and returns a pending future.
If any error occurs when writing immediately or asynchronously, the
`abort()` method is called.
The retured ``Future`` is stored in the ``self.pending`` dictionary
keyed on correlation id, so that clients can say
``response = yield conn.send(message)``
and expect the correctly correlated response (or a raised exception)
regardless of when the broker responds.
"""
f = concurrent.Future()
if self.closing:
f.set_exception(BrokerConnectionError(self.host, self.port))
return f
payload = message.serialize()
payload = size_struct.pack(len(payload)) + payload
self.api_correlation[message.correlation_id] = message.api
self.pending[message.correlation_id] = f
def handle_write(write_future):
with self.socket_error_handling("Error writing to socket."):
write_future.result()
with self.socket_error_handling("Error writing to socket."):
self.stream.write(payload).add_done_callback(handle_write)
return f
def test_fetch_future_exception(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
future = concurrent.Future()
fetch.return_value = future
future.set_exception(exceptions.DynamoDBException())
with self.assertRaises(exceptions.DynamoDBException):
yield self.client.create_table(self.generic_table_definition())
def test_serve(self):
f = concurrent.Future()
f.set_result(123)
self.mock_nats_client.subscribe_async.return_value = f
yield self.server.serve()
self.assertEquals([123], self.server._sub_ids)
def flush(self):
if not self.readonly:
data = self.data_to_send()
if data:
return self.stream.write(data)
future = concurrent.Future()
future.set_result(None)
return future
def future_error(self, exc, instance=None, tb=None):
f = concurrent.Future()
if instance is None or tb is None:
f.set_exception(exc)
else:
f.set_exc_info((exc, instance, tb))
return f
def fetch(self, request, *args, **kwargs):
future = concurrent.Future()
future.set_result(
httpclient.HTTPResponse(
request=request,
code=500,
error=ValueError('Expected programming error')))
return future
def test_dlx(self):
suffix = self.get_random_name()
routing_key = "%s_routing_key" % suffix
dlx_routing_key = "%s_dlx_routing_key" % suffix
channel = yield self.create_channel()
f = concurrent.Future()
@gen.coroutine
def dlx_handle(message):
message.ack()
self.assertEqual(message.body, body)
self.assertEqual(message.routing_key, dlx_routing_key)
f.set_result(True)
direct_exchange = yield self.declare_exchange(
'direct', channel=channel, auto_delete=True) # type:
# topika.Exchange
dlx_exchange = yield channel.declare_exchange('dlx',
ExchangeType.DIRECT,
auto_delete=True)
direct_queue = yield channel.declare_queue(
"%s_direct_queue" % suffix,
auto_delete=True,
arguments={
'x-message-ttl': 300,
'x-dead-letter-exchange': 'dlx',
'x-dead-letter-routing-key': dlx_routing_key
})
dlx_queue = yield channel.declare_queue("%s_dlx_queue" % suffix,
auto_delete=True)
yield dlx_queue.consume(dlx_handle)
yield dlx_queue.bind(dlx_exchange, dlx_routing_key)
yield direct_queue.bind(direct_exchange, routing_key)
body = bytes(shortuuid.uuid(), 'utf-8')
try:
yield direct_exchange.publish(
Message(body,
content_type='text/plain',
headers={
'x-message-ttl': 100,
'x-dead-letter-exchange': 'dlx',
}), routing_key)
if not f.done():
yield f
finally:
yield dlx_queue.unbind(dlx_exchange, routing_key)
yield direct_queue.unbind(direct_exchange, routing_key)
yield direct_queue.delete()
yield direct_exchange.delete()
yield dlx_exchange.delete()
