async def game(self):
""" Run the game. """
await self.send_sentence()
checkpoint = time_started = datetime.now()
timeout = self.calculate_timeout()
# We'll wait for a message from all of our participants
for i in range(len(self.participants)):
reply = await client.wait_for_message(timeout=timeout, channel=self.channel, check=self.is_participant)
if not reply:
await client.send_message(self.channel, "**Time is up.**")
return
# Delete the member's reply in order to avoid cheating
asyncio.ensure_future(client.delete_message(reply))
now = datetime.now()
# Calculate the time elapsed since the game started
time_elapsed = (now - time_started).total_seconds()
# Calculate the accuracy, wpm and send the message
accuracy = self.calculate_accuracy(reply.clean_content)
wpm = self.calculate_wpm(time_elapsed)
m = self.reply.format(member=reply.author, time=time_elapsed, wpm=wpm, accuracy=accuracy)
asyncio.ensure_future(client.send_message(self.channel, m))
# Reduce the timeout by the current time elapsed and create a checkpoint for the next timeout calculation
timeout -= int((now - checkpoint).total_seconds())
checkpoint = now
await asyncio.sleep(1)
await client.send_message(self.channel, "**Everyone finished!**")
async def handle_client(reader, writer, loop=None):
header_str = ''
payload = b''
try:
while True:
line = await reader.readline()
if not line or line == b'\r\n':
break
header_str += line.decode('utf-8')
match = length_regex.search(header_str)
if match:
length = int(match.group(1))
while len(payload) < length:
payload += await reader.read(BUFFER_SIZE)
headers = header_str.split('\r\n')[:-1]
method, path, version = headers[0].split(' ')
if method == 'CONNECT': # HTTPS
host, port_str = path.split(':')
port = int(port_str)
rreader, rwriter = await asyncio.open_connection(host=host, port=port, loop=loop)
writer.write(b'HTTP/1.1 200 Connection Established\r\n\r\n')
tasks = [
asyncio.ensure_future(relay(reader, rwriter), loop=loop),
asyncio.ensure_future(relay(rreader, writer), loop=loop)
]
await asyncio.wait(tasks, loop=loop)
else: # HTTP
r = urlparse(path)
host = r.hostname
port = r.port or 80
new_headers = []
new_headers.append(' '.join([method, r.path, version]))
connection_header_found = False
for header in headers[1:]:
name, value = header.strip().split(': ', 1)
if name.lower() == 'proxy-connection':
continue
elif name.lower() == 'connection':
connection_header_found = True
new_headers.append('Connection: close')
else:
new_headers.append(header)
if not connection_header_found:
new_headers.append('Connection: close')
new_headers.append('\r\n')
new_header_str = '\r\n'.join(new_headers)
rreader, rwriter = await asyncio.open_connection(host=host, port=port, loop=loop)
rwriter.write(new_header_str.encode('utf-8'))
if payload:
rwriter.write(payload)
await rwriter.drain()
await relay(rreader, writer)
except:
traceback.print_exc()
finally:
writer.close()
async def get_directly_linked_peers(
request, event_loop,
peer1_class=LESPeer, peer1_chaindb=None,
peer2_class=LESPeer, peer2_chaindb=None):
"""Create two peers with their readers/writers connected directly.
The first peer's reader will write directly to the second's writer, and vice-versa.
"""
peer1, peer2 = await get_directly_linked_peers_without_handshake(
peer1_class, peer1_chaindb,
peer2_class, peer2_chaindb)
# Perform the base protocol (P2P) handshake.
await asyncio.gather(peer1.do_p2p_handshake(), peer2.do_p2p_handshake())
assert peer1.sub_proto.name == peer2.sub_proto.name
assert peer1.sub_proto.version == peer2.sub_proto.version
assert peer1.sub_proto.cmd_id_offset == peer2.sub_proto.cmd_id_offset
# Perform the handshake for the enabled sub-protocol.
await asyncio.gather(peer1.do_sub_proto_handshake(), peer2.do_sub_proto_handshake())
asyncio.ensure_future(peer1.run())
asyncio.ensure_future(peer2.run())
def finalizer():
async def afinalizer():
await peer1.stop()
await peer2.stop()
event_loop.run_until_complete(afinalizer())
request.addfinalizer(finalizer)
return peer1, peer2
async def get_participants(self):
""" Wait for input and get all participants. """
for i in range(self.num):
def check(m):
if m.content.lower().strip() == "i" and m.author not in self.participants:
return True
return False
# Wait with a timeout of 2 minutes and check each message with check(m)
reply = await client.wait_for_message(timeout=120, channel=self.channel, check=check)
if reply: # A user replied with a valid check
asyncio.ensure_future(
client.say(self.message,
"{} has entered! `{}/{}`. Type `I` to join!".format(
reply.author.mention, i + 1, self.num))
)
self.participants.append(reply.author)
# Remove the message if bot has permissions
if self.member.permissions_in(self.channel).manage_messages:
asyncio.ensure_future(client.delete_message(reply))
else:
# At this point we got no reply in time and thus, gathering participants failed
await client.say(self.message, "**The {} game failed to gather {} participants.**".format(
self.name, self.num))
started.pop(started.index(self.channel.id))
return False
return True
def _start_timeout(self) -> None:
"""
Start auto flush timeout. Similar to Vim's `timeoutlen` option.
Start a background thread with a timer. When this timeout expires and
no key was pressed in the meantime, we flush all data in the queue and
call the appropriate key binding handlers.
"""
timeout = get_app().timeoutlen
if timeout is None:
return
counter = self._keys_pressed
async def wait() -> None:
" Wait for timeout. "
await sleep(timeout)
if len(self.key_buffer) > 0 and counter == self._keys_pressed:
# (No keys pressed in the meantime.)
flush_keys()
def flush_keys() -> None:
" Flush keys. "
self.feed(_Flush)
self.process_keys()
# Automatically flush keys.
# (_daemon needs to be set, otherwise, this will hang the
# application for .5 seconds before exiting.)
ensure_future(wait())
async def STUN(self, peer):
"""
Perform a STUN sequence between self and peer
:param peer:
:return: (own_addr, remote_addr) | None
"""
own_addr = asyncio.ensure_future(self.connectivity.ProbePeerNAT(peer))
remote_addr = asyncio.ensure_future(peer.connectivity.ProbePeerNAT(self))
(done, pending) = await asyncio.wait([own_addr, remote_addr], return_when=asyncio.FIRST_COMPLETED)
if own_addr.done() and remote_addr.done() and not own_addr.cancelled() and not remote_addr.cancelled():
# Both peers got it the first time
return own_addr.result(), remote_addr.result()
if own_addr.done() and not own_addr.cancelled():
# Remote received our packet, we didn't receive theirs
# Instruct remote to try our new address
own_addr = own_addr.result()
remote_addr = await peer.connectivity.ProbePeerNAT(self, use_address=own_addr)
elif remote_addr.done() and not remote_addr.cancelled():
# Opposite of the above
remote_addr = remote_addr.result()
own_addr = await self.connectivity.ProbePeerNAT(peer, use_address=remote_addr)
for p in pending:
if not p.done():
p.cancel()
return own_addr, remote_addr
async def handle_game_state(self, state):
"""
Changes in game state
:param state: new state
:return: None
"""
if state == 'Idle':
await self._handle_idle_state()
self._mark_dirty()
elif state == 'Lobby':
# The game is initialized and awaiting commands
# At this point, it is listening locally on the
# port we told it to (self.player.game_port)
# We schedule an async task to determine their connectivity
# and respond appropriately
#
# We do not yield from the task, since we
# need to keep processing other commands while it runs
asyncio.ensure_future(self._handle_lobby_state())
elif state == 'Launching':
if self.player.state == PlayerState.HOSTING:
await self.game.launch()
if len(self.game.mods.keys()) > 0:
async with db.db_pool.get() as conn:
cursor = await conn.cursor()
uids = list(self.game.mods.keys())
await cursor.execute("UPDATE mod_stats s "
"JOIN mod_version v ON v.mod_id = s.mod_id "
"SET s.times_played = s.times_played + 1 WHERE v.uid in %s", (uids,))
elif state == 'Ended':
await self.on_connection_lost()
def make_room(self):
voice_channel = None
#get the Discord voice channel
for channel in self._server.channels:
if channel.type == discord.ChannelType.voice and channel.name == config.VOICE_CHANNEL_NAME:
voice_channel = channel
if not voice_channel:
yield from self._write_error('Could not find the voice channel "{0}"'.format(config.VOICE_CHANNEL_NAME))
return
channel_name = self.get_race_channel_name(voice_channel)
#we now have the voice channel. create a new room and set its permissions
new_channel = yield from self._client.create_channel(self._server, channel_name)
if new_channel:
self._room_list.append(new_channel)
asyncio.ensure_future(self.set_raceroom_permissions(new_channel, voice_channel))
name_list_string = ''
for user in self.get_non_admins(voice_channel):
name_list_string += ' ' + user.mention + ','
self._write('Made race channel {0} for{1}.'.format(new_channel.mention, name_list_string[:-1]))
async def _handle_lobby_state(self):
"""
The game has told us it is ready and listening on
self.player.game_port for UDP.
We determine the connectivity of the peer and respond
appropriately
"""
try:
player_state = self.player.state
if player_state == PlayerState.HOSTING:
self.send_HostGame(self.game.map_folder_name)
# If the player is joining, we connect him to host
# followed by the rest of the players.
elif player_state == PlayerState.JOINING:
await self.ConnectToHost(self.game.host.game_connection)
if self._state is GameConnectionState.ENDED: # We aborted while trying to connect
return
self._state = GameConnectionState.CONNECTED_TO_HOST
self.game.add_game_connection(self)
for peer in self.game.connections:
if peer != self and peer.player != self.game.host:
self.log.debug("%s connecting to %s", self.player, peer)
asyncio.ensure_future(self.ConnectToPeer(peer))
except Exception as e:
self.log.exception(e)
async def main():
if os.path.exists(socket):
os.unlink(socket)
server = await asyncio.start_unix_server(on_unix_connect, socket)
asyncio.ensure_future(i3_event_listener())
await server.wait_closed()
async def bond(self, node: Node) -> bool:
"""Bond with the given node.
Bonding consists of pinging the node, waiting for a pong and maybe a ping as well.
It is necessary to do this at least once before we send find_node requests to a node.
"""
if node in self.routing:
return True
pingid = self.ping(node)
got_pong = await self.wait_pong(pingid)
if not got_pong:
self.logger.debug("bonding failed, didn't receive pong from %s", node)
# Drop the failing node and schedule a populate_not_full_buckets() call to try and
# fill its spot.
self.routing.remove_node(node)
asyncio.ensure_future(self.populate_not_full_buckets())
return False
# Give the remote node a chance to ping us before we move on and start sending find_node
# requests. It is ok for wait_ping() to timeout and return false here as that just means
# the remote remembers us.
await self.wait_ping(node)
self.logger.debug("bonding completed successfully with %s", node)
self.update_routing_table(node)
return True
def __init__(self, explorer, exp_manager, experiment_db_ctl):
QtWidgets.QDialog.__init__(self, parent=explorer)
self.resize(710, 700)
self.setWindowTitle("Open file outside repository")
self.explorer = explorer
self.exp_manager = exp_manager
self.experiment_db_ctl = experiment_db_ctl
grid = QtWidgets.QGridLayout()
self.setLayout(grid)
grid.addWidget(QtWidgets.QLabel("Location:"), 0, 0)
self.location_label = QtWidgets.QLabel("")
grid.addWidget(self.location_label, 0, 1)
grid.setColumnStretch(1, 1)
self.file_list = QtWidgets.QListWidget()
asyncio.ensure_future(self.refresh_view())
grid.addWidget(self.file_list, 1, 0, 1, 2)
self.file_list.doubleClicked.connect(self.accept)
buttons = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel)
grid.addWidget(buttons, 2, 0, 1, 2)
buttons.accepted.connect(self.accept)
buttons.rejected.connect(self.reject)
def web_sockets_handler(self, websocket, path):
self.__connected.add(websocket)
print('Connected: %s' % websocket)
try:
yield from asyncio.wait([ws.send('client connected') for ws in self.__connected])
while True:
listener_task = asyncio.ensure_future(websocket.recv())
producer_task = asyncio.ensure_future(self.producer())
done, pending = yield from asyncio.wait(
[listener_task, producer_task],
return_when=asyncio.FIRST_COMPLETED)
if listener_task in done:
message = listener_task.result()
yield from self.consumer(message)
else:
listener_task.cancel()
if producer_task in done:
message = producer_task.result()
yield from websocket.send(message)
else:
producer_task.cancel()
except websockets.exceptions.ConnectionClosed:
print('Connection %s closed' % websocket)
finally:
self.__connected.remove(websocket)
def command_upload(self, args):
if args is None:
self.core.command.help('upload')
return
filename, = args
filename = expanduser(filename)
asyncio.ensure_future(self.async_upload(filename))
async def handle(self) :
msg = await self.zmqSubSocket.recv_multipart()
topic = msg[0]
body = msg[1]
sequence = "Unknown"
if len(msg[-1]) == 4:
msgSequence = struct.unpack('<I', msg[-1])[-1]
sequence = str(msgSequence)
if topic == b"hashblock":
print('- HASH BLOCK ('+sequence+') -')
print(binascii.hexlify(body))
elif topic == b"hashtx":
print('- HASH TX ('+sequence+') -')
print(binascii.hexlify(body))
elif topic == b"rawblock":
print('- RAW BLOCK HEADER ('+sequence+') -')
print(binascii.hexlify(body[:80]))
elif topic == b"rawtx":
print('- RAW TX ('+sequence+') -')
print(binascii.hexlify(body))
elif topic == "hashgovernancevote":
print('- HASH GOVERNANCE VOTE ('+sequence+') -')
print(binascii.hexlify(body))
elif topic == "hashgovernanceobject":
print('- HASH GOVERNANCE OBJECT ('+sequence+') -')
print(binascii.hexlify(body))
elif topic == "rawgovernancevote":
print('- RAW GOVERNANCE VOTE ('+sequence+') -')
print(binascii.hexlify(body))
elif topic == "rawgovernanceobject":
print('- RAW GOVERNANCE OBJECT ('+sequence+') -')
print(binascii.hexlify(body))
# schedule ourselves to receive the next message
asyncio.ensure_future(self.handle())
def on_receive_append_entries_response(self, data):
sender_id = data['sender']
# logging.debug('append_entries_response %s from %s', data, sender_id)
# Count all unique responses per particular heartbeat interval
# and step down via <step_down_timer> if leader doesn't get majority of
# responses for <step_down_missed_heartbeats> heartbeats
# import ipdb; ipdb.set_trace()
if data['request_id'] in self.response_map:
self.response_map[data['request_id']].add(sender_id)
answered = len(self.response_map[data['request_id']])
if self.server.is_majority(answered + 1):
# logging.debug('leader %s step down reset', self.server.id)
self.step_down_timer.reset()
del self.response_map[data['request_id']]
if not data['success']:
self.log.next_index[sender_id] = \
max(self.log.next_index[sender_id] - 1, 1)
else:
self.log.next_index[sender_id] = data['last_log_index'] + 1
self.log.match_index[sender_id] = data['last_log_index']
self.update_commit_index()
# Send AppendEntries RPC to continue updating fast-forward log
# (data['success'] == False) or in case there are new entries to sync
# (data['success'] == data['updated'] == True)
if self.log.last_log_index >= self.log.next_index[sender_id]:
host, port = sender_id.split(':')
addr = (host.strip(), port.strip())
asyncio.ensure_future(self.append_entries(destination=addr),
loop=self.server.loop)
def welcome_if_new(self, node):
"""
Given a new node, send it all the keys/values it should be storing,
then add it to the routing table.
@param node: A new node that just joined (or that we just found out
about).
Process:
For each key in storage, get k closest nodes. If newnode is closer
than the furtherst in that list, and the node for this server
is closer than the closest in that list, then store the key/value
on the new node (per section 2.5 of the paper)
"""
if not self.router.is_new_node(node):
return
log.info("never seen %s before, adding to router", node)
for key, value in self.storage:
keynode = Node(digest(key))
neighbors = self.router.find_neighbors(keynode)
if neighbors:
last = neighbors[-1].distance_to(keynode)
new_node_close = node.distance_to(keynode) < last
first = neighbors[0].distance_to(keynode)
this_closest = self.source_node.distance_to(keynode) < first
if not neighbors or (new_node_close and this_closest):
asyncio.ensure_future(self.call_store(node, key, value))
self.router.add_contact(node)
async def wait_for_cpr_responses(self, timeout: int = 1) -> None:
"""
Wait for a CPR response.
"""
cpr_futures = list(self._waiting_for_cpr_futures) # Make copy.
# When there are no CPRs in the queue. Don't do anything.
if not cpr_futures or self.cpr_support == CPR_Support.NOT_SUPPORTED:
return None
async def wait_for_responses() -> None:
for response_f in cpr_futures:
await response_f
async def wait_for_timeout() -> None:
await sleep(timeout)
# Got timeout, erase queue.
self._waiting_for_cpr_futures = deque()
futures = [
ensure_future(wait_for_responses()),
ensure_future(wait_for_timeout()),
]
await wait(futures, return_when=FIRST_COMPLETED)
def on_receive_request_vote(self, data):
if self.voted_for is None and not data['type'].endswith('_response'):
# If the logs have last entries with different terms,
# then the log with the later term is more up-to-date.
# If the logs end with the same term, then whichever log is longer
# is more up-to-date.
# Candidates' log has to be up-to-date
if data['last_log_term'] != self.log.last_log_term:
up_to_date = data['last_log_term'] > self.log.last_log_term
else:
up_to_date = data['last_log_index'] >= self.log.last_log_index
if up_to_date:
self.voted_for = data['candidate_id']
response = {
'type': 'request_vote_response',
'term': self.term,
'vote_granted': up_to_date
}
asyncio.ensure_future(self.server.send(response, data['sender']),
loop=self.server.loop)
async def test_connect(application, fake_server_with_blockchain, bob):
await BlockchainProcessor.instanciate(application).initialize_blockchain(application.currency)
connection_config_dialog = ConnectionConfigController.create_connection(None, application)
def close_dialog():
if connection_config_dialog.view.isVisible():
connection_config_dialog.view.close()
async def exec_test():
QTest.mouseClick(connection_config_dialog.view.button_connect, Qt.LeftButton)
await asyncio.sleep(0.6)
QTest.mouseClick(connection_config_dialog.view.button_accept, Qt.LeftButton)
await asyncio.sleep(0.1)
assert connection_config_dialog.view.stacked_pages.currentWidget() == connection_config_dialog.view.page_connection
assert_key_parameters_behaviour(connection_config_dialog, bob)
QTest.mouseClick(connection_config_dialog.view.button_next, Qt.LeftButton)
await asyncio.sleep(0.1)
assert connection_config_dialog.view.stacked_pages.currentWidget() == connection_config_dialog.view.page_services
assert len(ConnectionsProcessor.instanciate(application).connections()) == 1
application.loop.call_later(10, close_dialog)
asyncio.ensure_future(exec_test())
await connection_config_dialog.async_exec()
await fake_server_with_blockchain.close()
async def handler(self, websocket, path):
x = 0
while True:
check_first_click = await websocket.recv()
check_first_click = json.loads(check_first_click)
if check_first_click['start']:
self.velocity1 = check_first_click['velocity1']
self.velocity2 = check_first_click['velocity2']
self.ramp = check_first_click['ramp']
self.dwell = check_first_click['dwell']
self.limit_lower = check_first_click['lower']
self.limit_upper = check_first_click['upper']
sfr_set_regbit(usb, ior['TRISA'], 5, 1)
self.check_port()
while True:
tasks = [asyncio.ensure_future(websocket.recv()), asyncio.ensure_future(self.set_graph_data(x))]
done, panding = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
if tasks[0] in done:
if json.loads(tasks[0].result())['start'] is False:
sfr_set_regbit(usb, ior['TRISA'], 5, 0)
break
else:
tasks[0].cancel()
if tasks[1] in done:
mess = tasks[1].result()
await websocket.send(str(mess))
await asyncio.sleep(0.1)
else:
tasks[1].cancel()
x += 1
async def extract_info(self, loop, *args, on_error=None, retry_on_error=False, **kwargs):
"""
Runs ytdl.extract_info within the threadpool. Returns a future that will fire when it's done.
If `on_error` is passed and an exception is raised, the exception will be caught and passed to
on_error as an argument.
"""
if callable(on_error):
try:
return await loop.run_in_executor(self.thread_pool, functools.partial(self.unsafe_ytdl.extract_info, *args, **kwargs))
except Exception as e:
# (youtube_dl.utils.ExtractorError, youtube_dl.utils.DownloadError)
# I hope I don't have to deal with ContentTooShortError's
if asyncio.iscoroutinefunction(on_error):
asyncio.ensure_future(on_error(e), loop=loop)
elif asyncio.iscoroutine(on_error):
asyncio.ensure_future(on_error, loop=loop)
else:
loop.call_soon_threadsafe(on_error, e)
if retry_on_error:
return await self.safe_extract_info(loop, *args, **kwargs)
else:
return await loop.run_in_executor(self.thread_pool, functools.partial(self.unsafe_ytdl.extract_info, *args, **kwargs))
async def read_from_server(host: str, port: int, loop):
# Exponential backoff
timeout = 0
while True:
try:
logging.info('Sleeping %s s', timeout)
await asyncio.sleep(timeout)
(reader, writer) = await asyncio.open_connection(host=host, port=port, loop=loop)
timeout = 0 # Reset timeout after successful connection
logging.info('Connected to %s:%s', host, port)
try:
while True:
pkt = await reader.readexactly(2)
len = struct.unpack('!H', pkt)[0]
pkt = pickle.loads(zlib.decompress(await reader.readexactly(len)))
asyncio.ensure_future(put_to_queue(pkt))
except ConnectionResetError:
logging.warning("Connection closed (%s:%s)", host, port)
except asyncio.IncompleteReadError:
logging.warning("Could not complete read (%s:%s)", host, port)
except ConnectionRefusedError:
logging.warning('Connection refused (%s:%s)', host, port)
except OSError: # Multiple connection refued at the same time (ipv4/6)
logging.warning('Connection refused (%s:%s)', host, port)
finally:
if timeout == 0:
timeout = 0.25
timeout *= 2 # double our waiting time
timeout = min(timeout, 30) # clamp to 30 s
def receive_message(self, msg):
if isinstance(msg, Response):
if msg.cseq in self._msgs[msg.method]:
original_msg = self._msgs[msg.method].get(msg.cseq)
transaction = self._pending[msg.method].get(msg.cseq)
transaction.feed_message(msg, original_msg=original_msg)
else:
raise ValueError('This Response SIP message doesn\'t have Request: "%s"' % msg)
else:
if msg.method != 'ACK':
hdrs = CIMultiDict()
hdrs['Via'] = msg.headers['Via']
hdrs['CSeq'] = msg.headers['CSeq']
hdrs['Call-ID'] = msg.headers['Call-ID']
self.send_reply(status_code=200,
status_message='OK',
to_details=msg.to_details,
from_details=msg.from_details,
headers=hdrs,
payload=None)
for callback_info in self.callbacks[msg.method.upper()]:
if asyncio.iscoroutinefunction(callback_info['callable']):
fut = callback_info['callable'](*((self, msg,) + callback_info['args']), **callback_info['kwargs'])
asyncio.ensure_future(fut)
else:
self.loop.call_soon(partial(callback_info['callable'], *((self, msg,) + callback_info['args']), **callback_info['kwargs']))
def main():
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('remote_server', type=str, help='The server to connect to')
parser.add_argument('remote_port', type=int, help='The port', default=8888)
parser.add_argument('--host', type=str, help='Listen IP', default='')
parser.add_argument('--port', type=int, help='The port', default=8765)
parser.add_argument('--log', type=str, help='log level', default='INFO')
args = parser.parse_args()
numeric_level = getattr(logging, args.log.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % args.log)
logging.basicConfig(level=numeric_level)
loop = asyncio.get_event_loop()
ws_server = loop.run_until_complete(websockets.serve(handle_ws, args.host, args.port, loop=loop))
asyncio.ensure_future(read_from_server(args.remote_server, args.remote_port, loop))
loop.call_later(60, gather_stats, loop)
for sock in ws_server.server.sockets:
addr = sock.getsockname()
logging.info('Listening for Websocket connections on %s:%s', addr[0], addr[1])
loop.run_forever()
loop.close()
return 0
def test_get_request(self):
@self.app.add_handler("/get_test")
class TestHandler(futurefinity.web.RequestHandler):
async def get(self, *args, **kwargs):
return self.get_link_arg("content")
server = self.app.listen(8888)
async def get_requests_result(self):
try:
self.requests_result = await self.loop.run_in_executor(
None, functools.partial(
requests.get,
("http://127.0.0.1:8888/get_test"
"?content=Hello, World!")
)
)
except:
traceback.print_exc()
finally:
server.close()
await server.wait_closed()
self.loop.stop()
asyncio.ensure_future(get_requests_result(self))
self.loop.run_forever()
self.assertEqual(self.requests_result.status_code, 200,
"Wrong Status Code")
self.assertEqual(self.requests_result.text, "Hello, World!")
def send_to_game(cls, message):
"""
Broadcast a message on the server. Make sure it isn't coming from the
bot (or else we get duplicate messages).
:param message: The message packet.
:return: Null
"""
nick = message.author.display_name
text = message.clean_content
server = message.server
if message.author.id != cls.client_id:
if message.content[0] == ".":
asyncio.ensure_future(cls.handle_command(message.content[1:]))
else:
for emote in server.emojis:
text = text.replace("<:{}:{}>".format(emote.name,emote.id),
":{}:".format(emote.name))
yield from cls.factory.broadcast("<^orange;Discord^reset;> <{}> {}"
"".format(nick, text),
mode=ChatReceiveMode.BROADCAST)
if cls.config.get_plugin_config(cls.name)["log_discord"]:
cls.logger.info("<{}> {}".format(nick, text))
if link_plugin_if_available(cls, "irc_bot"):
asyncio.ensure_future(cls.plugins['irc_bot'].bot_write(
"<DC><{}> {}".format(nick, text)))
def release(self, conn):
"""Returns used connection back into pool.
When returned connection has db index that differs from one in pool
the connection will be closed and dropped.
When queue of free connections is full the connection will be dropped.
"""
assert conn in self._used, (
"Invalid connection, maybe from other pool", conn)
self._used.remove(conn)
if not conn.closed:
if conn.in_transaction:
logger.warning(
"Connection %r is in transaction, closing it.", conn)
conn.close()
elif conn.in_pubsub:
logger.warning(
"Connection %r is in subscribe mode, closing it.", conn)
conn.close()
elif conn._waiters:
logger.warning(
"Connection %r has pending commands, closing it.", conn)
conn.close()
elif conn.db == self.db:
if self.maxsize and self.freesize < self.maxsize:
self._pool.append(conn)
else:
# consider this connection as old and close it.
conn.close()
else:
conn.close()
# FIXME: check event loop is not closed
asyncio.ensure_future(self._wakeup(), loop=self._loop)
def test_empty_wot_tab(self):
wot_tab = WotTabWidget(self.application)
future = asyncio.Future()
def open_widget():
wot_tab.widget.show()
return future
async def async_open_widget():
srv, port, url = await self.mock_nice_blockchain.create_server()
self.addCleanup(srv.close)
await open_widget()
def close_dialog():
if wot_tab.widget.isVisible():
wot_tab.widget.close()
future.set_result(True)
async def exec_test():
await asyncio.sleep(1)
self.assertTrue(wot_tab.widget.isVisible())
self.lp.call_soon(close_dialog)
asyncio.ensure_future(exec_test())
self.lp.call_later(15, close_dialog)
self.lp.run_until_complete(async_open_widget())
async def test_register_empty_blockchain(application, fake_server, bob, tmpdir):
tmpdir.mkdir("test_register")
revocation_file = tmpdir.join("test_register").join("revocation.txt")
identity_file = tmpdir.join("test_register").join("identity.txt")
await BlockchainProcessor.instanciate(application).initialize_blockchain(application.currency)
connection_config_dialog = ConnectionConfigController.create_connection(None, application)
def close_dialog():
if connection_config_dialog.view.isVisible():
connection_config_dialog.view.close()
async def exec_test():
QTest.mouseClick(connection_config_dialog.view.button_register, Qt.LeftButton)
await asyncio.sleep(0.6)
QTest.mouseClick(connection_config_dialog.view.button_accept, Qt.LeftButton)
await asyncio.sleep(0.1)
assert connection_config_dialog.view.stacked_pages.currentWidget() == connection_config_dialog.view.page_connection
assert_key_parameters_behaviour(connection_config_dialog, bob)
QTest.mouseClick(connection_config_dialog.view.button_next, Qt.LeftButton)
connection_config_dialog.model.connection.password = bob.password
await asyncio.sleep(1)
select_file_dialog(str(revocation_file))
await asyncio.sleep(1)
await asyncio.sleep(1)
revocation_file.ensure()
assert connection_config_dialog.view.stacked_pages.currentWidget() == connection_config_dialog.view.page_services
assert len(ConnectionsProcessor.instanciate(application).connections()) == 1
accept_dialog("Registration")
application.loop.call_later(30, close_dialog)
asyncio.ensure_future(exec_test())
await connection_config_dialog.async_exec()
await fake_server.close()
def schedule_broadcast_sys_topic(self, topic_basename, data):
return asyncio.ensure_future(
self._broadcast_sys_topic(DOLLAR_SYS_ROOT + topic_basename, data),
loop=self.context.loop,
)
async def setup_learner():
await download_file(export_file_url, path / export_file_name)
try:
learn = load_learner(path, export_file_name)
return learn
except RuntimeError as e:
if len(e.args) > 0 and 'CPU-only machine' in e.args[0]:
print(e)
message = "\n\nThis model was trained with an old version of fastai and will not work in a CPU environment.\n\nPlease update the fastai library in your training environment and export your model again.\n\nSee instructions for 'Returning to work' at https://course.fast.ai."
raise RuntimeError(message)
else:
raise
loop = asyncio.get_event_loop()
tasks = [asyncio.ensure_future(setup_learner())]
learn = loop.run_until_complete(asyncio.gather(*tasks))[0]
loop.close()
@app.route('/')
async def homepage(request):
html_file = path / 'view' / 'index.html'
return HTMLResponse(html_file.open().read())
@app.route('/analyze', methods=['POST'])
async def analyze(request):
img_data = await request.form()
img_bytes = await (img_data['file'].read())
img = open_image(BytesIO(img_bytes))
# -------------------------------------------------------------------------------
async def get_corouting():
print("run get corouting...")
await asyncio.sleep(2)
return "now time %s" % time.time()
# 回调函数，参数必须是future
def cb(future):
print("future: ", future)
print(future.result())
loop = asyncio.get_event_loop()
# 传入 协程 对象，构建任务对象
future = asyncio.ensure_future(get_corouting())
# 可以添加回调参数
# future.add_done_callback(cb)
# 传入 任务对象，到事件循环中可直接运行
result = loop.run_until_complete(future)
print("result: ", result)
print("result: ", future.result())
# output:
# ---------------------------------------------------------------------------
# run get corouting...
# result: now time 1602313682.7316842
# result: now time 1602313682.7316842
def __init__(self, curr_man):
self._curr_man = curr_man
self._task = asyncio.ensure_future(self._job())
async def start(self):
""" Start GUI and event loop """
self._curses_init()
self.gui_loop = asyncio.ensure_future(self.worker())
# coding= utf-8
import time
import asyncio
async def run(x):
print('waiting:%d' % x)
return "done %d" % x
# 定义个回调函数
def callback(future):
print("callback", future.result())
start = time.time()
corountine = run(2)
loop = asyncio.get_event_loop()
task = asyncio.ensure_future(corountine)
# 给任务添加回调，在任务结束后调用回调函数
task.add_done_callback(callback)
loop.run_until_complete(task)
end = time.time()
print("time", end - start)
from asyncio import ensure_future, get_event_loop
from pyipv8.ipv8.configuration import get_default_configuration
from pyipv8.ipv8_service import IPv8
async def start_ipv8():
# The first IPv8 will attempt to claim a port.
await IPv8(get_default_configuration()).start()
# The second IPv8 will attempt to claim a port.
# It cannot claim the same port and will end up claiming a different one.
await IPv8(get_default_configuration()).start()
ensure_future(start_ipv8())
get_event_loop().run_forever()
async def handle_websocket(downstream_request, upstream_headers,
upstream_url):
protocol = downstream_request.headers.get('Sec-WebSocket-Protocol')
protocols = (protocol, ) if protocol else ()
async def proxy_msg(msg, to_ws):
if msg.type == aiohttp.WSMsgType.TEXT:
await to_ws.send_str(msg.data)
elif msg.type == aiohttp.WSMsgType.BINARY:
await to_ws.send_bytes(msg.data)
elif msg.type == aiohttp.WSMsgType.CLOSE:
await to_ws.close()
elif msg.type == aiohttp.WSMsgType.ERROR:
await to_ws.close()
async def upstream():
try:
async with client_session.ws_connect(
str(upstream_url),
headers=upstream_headers,
protocols=protocols) as upstream_ws:
upstream_connection.set_result(upstream_ws)
downstream_ws = await downstream_connection
async for msg in upstream_ws:
await proxy_msg(msg, downstream_ws)
except BaseException as exception:
if not upstream_connection.done():
upstream_connection.set_exception(exception)
raise
finally:
try:
await downstream_ws.close()
except UnboundLocalError:
# If we didn't get to the line that creates `downstream_ws`
pass
# This is slightly convoluted, but aiohttp documents that reading
# from websockets should be done in the same task as the websocket was
# created, so we read from downstream in _this_ task, and create
# another task to connect to and read from the upstream socket. We
# also need to make sure we wait for each connection before sending
# data to it
downstream_connection = asyncio.Future()
upstream_connection = asyncio.Future()
upstream_task = asyncio.ensure_future(upstream())
try:
upstream_ws = await upstream_connection
_, _, _, with_session_cookie = downstream_request[SESSION_KEY]
downstream_ws = await with_session_cookie(
web.WebSocketResponse(protocols=protocols, heartbeat=30))
await downstream_ws.prepare(downstream_request)
downstream_connection.set_result(downstream_ws)
async for msg in downstream_ws:
await proxy_msg(msg, upstream_ws)
finally:
upstream_task.cancel()
return downstream_ws
async def handle_comm(self, comm, shutting_down=shutting_down):
""" Dispatch new communications to coroutine-handlers
Handlers is a dictionary mapping operation names to functions or
coroutines.
{'get_data': get_data,
'ping': pingpong}
Coroutines should expect a single Comm object.
"""
if self.__stopped:
comm.abort()
return
address = comm.peer_address
op = None
logger.debug("Connection from %r to %s", address, type(self).__name__)
self._comms[comm] = op
try:
while True:
try:
msg = await comm.read()
logger.debug("Message from %r: %s", address, msg)
except EnvironmentError as e:
if not shutting_down():
logger.debug(
"Lost connection to %r while reading message: %s."
" Last operation: %s",
address,
e,
op,
)
break
except Exception as e:
logger.exception(e)
await comm.write(error_message(e, status="uncaught-error"))
continue
if not isinstance(msg, dict):
raise TypeError(
"Bad message type. Expected dict, got\n " + str(msg)
)
try:
op = msg.pop("op")
except KeyError:
raise ValueError(
"Received unexpected message without 'op' key: " + str(msg)
)
if self.counters is not None:
self.counters["op"].add(op)
self._comms[comm] = op
serializers = msg.pop("serializers", None)
close_desired = msg.pop("close", False)
reply = msg.pop("reply", True)
if op == "close":
if reply:
await comm.write("OK")
break
result = None
try:
if op in self.blocked_handlers:
_msg = (
"The '{op}' handler has been explicitly disallowed "
"in {obj}, possibly due to security concerns."
)
exc = ValueError(_msg.format(op=op, obj=type(self).__name__))
handler = raise_later(exc)
else:
handler = self.handlers[op]
except KeyError:
logger.warning(
"No handler %s found in %s",
op,
type(self).__name__,
exc_info=True,
)
else:
if serializers is not None and has_keyword(handler, "serializers"):
msg["serializers"] = serializers # add back in
logger.debug("Calling into handler %s", handler.__name__)
try:
result = handler(comm, **msg)
if hasattr(result, "__await__"):
result = asyncio.ensure_future(result)
self._ongoing_coroutines.add(result)
result = await result
except (CommClosedError, CancelledError) as e:
if self.status == "running":
logger.info("Lost connection to %r: %s", address, e)
break
except Exception as e:
logger.exception(e)
result = error_message(e, status="uncaught-error")
if reply and result != "dont-reply":
try:
await comm.write(result, serializers=serializers)
except (EnvironmentError, TypeError) as e:
logger.debug(
"Lost connection to %r while sending result for op %r: %s",
address,
op,
e,
)
break
msg = result = None
if close_desired:
await comm.close()
if comm.closed():
break
finally:
del self._comms[comm]
if not shutting_down() and not comm.closed():
try:
comm.abort()
except Exception as e:
logger.error(
"Failed while closing connection to %r: %s", address, e
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--bind", help="IP address to bind to", default="")
parser.add_argument("--port", help="TCP port", type=int, default=8000)
parser.add_argument("--level", help="start on level", type=int, default=1)
parser.add_argument("--seed", help="Seed number", type=int, default=0)
parser.add_argument(
"--timeout", help="Timeout after this amount of steps", type=int, default=TIMEOUT
)
parser.add_argument(
"--grading-server",
help="url of grading server",
default="http://bomberman-aulas.ws.atnog.av.it.pt/game",
)
args = parser.parse_args()
if args.seed > 0:
random.seed(args.seed)
g = GameServer(args.level, args.timeout, args.grading_server)
game_loop_task = asyncio.ensure_future(g.mainloop())
logger.info("Listenning @ %s:%s", args.bind, args.port)
websocket_server = websockets.serve(g.incomming_handler, args.bind, args.port)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(websocket_server, game_loop_task))
loop.close()
def __init__(self, timeout, callback, packet):
self._timeout = timeout
self._callback = callback
self.packet = packet
self.flag = 0
self._task = asyncio.ensure_future(self._job())
async def handle_application_http_spawning(
downstream_request,
method,
upstream_url,
query,
host_html_path,
host_api_url,
public_host,
):
host = downstream_request.headers['host']
try:
logger.info('Spawning: Attempting to connect to %s', upstream_url)
response = await handle_http(
downstream_request,
method,
CIMultiDict(application_headers(downstream_request)),
upstream_url,
query,
await get_data(downstream_request),
spawning_http_timeout,
# Although the application is spawning, if the response makes it back to the client,
# we know the application is running, so we return the _running_ CSP headers
(
(
'content-security-policy',
csp_application_running_direct(host, public_host),
), ),
)
except Exception: # pylint: disable=broad-except
logger.info('Spawning: Failed to connect to %s', upstream_url)
return await handle_http(
downstream_request,
'GET',
CIMultiDict(admin_headers_request(downstream_request)),
admin_root + host_html_path + '/spawning',
{},
b'',
default_http_timeout,
(('content-security-policy', csp_application_spawning), ),
)
else:
# Once a streaming response is done, if we have not yet returned
# from the handler, it looks like aiohttp can cancel the current
# task. We set RUNNING in another task to avoid it being cancelled
async def set_application_running():
async with client_session.request(
'PATCH',
host_api_url,
json={'state': 'RUNNING'},
headers=CIMultiDict(
admin_headers_request(downstream_request)),
timeout=default_http_timeout,
) as patch_response:
await patch_response.read()
asyncio.ensure_future(set_application_running())
await send_to_google_analytics(downstream_request)
return response
def cancel_task(self, task):
""" Cancels current task and sends order to await cancellation """
task.cancel()
asyncio.ensure_future(self.await_cancellation(task))
def taskRun(self):
self.__task = asyncio.ensure_future(self.follow())
def register_peer(self, peer: BasePeer) -> None:
asyncio.ensure_future(self.handle_peer(cast(ETHPeer, peer)))
async def count_valid_passports(inpt, fn):
tasks = []
for passport in inpt.split("\n\n"):
tasks.append(asyncio.ensure_future(fn(passport)))
return sum(await asyncio.gather(*tasks))
def reload_handler(loop, cam, signal, frame):
print("Reloading background / foreground images")
asyncio.ensure_future(cam.load_images())
def done_callback(fut):
asyncio.ensure_future(done())
def _resubscribe(self, sub):
self._subs[sub.channel] = sub
asyncio.ensure_future(self._subscribe([sub.channel]))
return sub
async def async_execute_pipeline(pipeline_id,
pipeline_steps,
pipeline_cwd,
trigger,
execution_id,
use_cache,
dependencies,
debug):
if debug:
logging.info("%s Async task starting", execution_id[:8])
ps = status_mgr().get(pipeline_id)
if not ps.start_execution(execution_id):
logging.info("%s START EXECUTION FAILED %s, BAILING OUT", execution_id[:8], pipeline_id)
return False, {}, []
ps.update_execution(execution_id, [])
if use_cache:
if debug:
logging.info("%s Searching for existing caches", execution_id[:8])
pipeline_steps = find_caches(pipeline_steps, pipeline_cwd)
execution_log = []
if debug:
logging.info("%s Building process chain:", execution_id[:8])
processes, stop_error_collecting = \
await construct_process_pipeline(pipeline_steps, pipeline_cwd, execution_log, debug)
processes[0].stdin.write(json.dumps(dependencies).encode('utf8') + b'\n')
processes[0].stdin.write(b'{"name": "_", "resources": []}\n')
processes[0].stdin.close()
def kill_all_processes():
for to_kill in processes:
try:
to_kill.kill()
except ProcessLookupError:
pass
success = True
pending = [asyncio.ensure_future(process_death_waiter(process))
for process in processes]
index_for_pid = dict(
(p.pid, i)
for i, p in enumerate(processes)
)
failed_index = None
while len(pending) > 0:
done = []
try:
done, pending = \
await asyncio.wait(pending,
return_when=asyncio.FIRST_COMPLETED,
timeout=10)
except CancelledError:
success = False
kill_all_processes()
for waiter in done:
process, return_code = waiter.result()
if return_code == 0:
if debug:
logging.info("%s DONE %s", execution_id[:8], process.args)
processes = [p for p in processes if p.pid != process.pid]
else:
if return_code > 0 and failed_index is None:
failed_index = index_for_pid[process.pid]
if debug:
logging.error("%s FAILED %s: %s", execution_id[:8], process.args, return_code)
success = False
kill_all_processes()
if success and not ps.update_execution(execution_id, execution_log):
logging.error("%s FAILED to update %s", execution_id[:8], pipeline_id)
success = False
kill_all_processes()
stats, error_log = await stop_error_collecting(failed_index)
if success is False:
stats = None
ps.update_execution(execution_id, execution_log, hooks=True)
ps.finish_execution(execution_id, success, stats, error_log)
logging.info("%s DONE %s %s %r", execution_id[:8], 'V' if success else 'X', pipeline_id, stats)
return success, stats, error_log
def connect(self):
self.status = STATUS_CONNECTING
success = yield from self._create_connection()
if not success:
asyncio.ensure_future(self.reconnect())
async def post(self):
"""Handle POST requests to the upload endpoint
Validate upload, get service name, create UUID, save to local storage,
then offload for async processing
---
description: Process Insights archive
responses:
202:
description: Upload payload accepted
413:
description: Payload too large
415:
description: Upload field not found
"""
mnm.uploads_total.inc()
self.request_id = self.request.headers.get('x-rh-insights-request-id', uuid.uuid4().hex)
self.account = "unknown"
# is this really ok to be optional?
self.b64_identity = self.request.headers.get('x-rh-identity')
if self.b64_identity:
with mnm.uploads_json_loads.labels(key="post").time():
header = json.loads(base64.b64decode(self.b64_identity))
self.identity = header['identity']
self.account = self.identity["account_number"]
extra = get_extra(account=self.account, request_id=self.request_id)
if not self.request.files.get('upload') and not self.request.files.get('file'):
return self.error(
415,
"Upload field not found",
files=list(self.request.files),
**extra
)
try:
upload_field = list(self.request.files)[0]
mnm.uploads_file_field.labels(field=upload_field).inc()
except IndexError:
pass
self.payload_data = self.request.files.get('upload')[0] if self.request.files.get('upload') else self.request.files.get('file')[0]
if self.request_id is None:
return self.error(400, "No request_id assigned. Upload failed.", **extra)
if self.upload_validation():
mnm.uploads_invalid.inc()
else:
mnm.uploads_valid.inc()
self.metadata = self.__get_metadata_from_request()
service_dict = get_service(self.payload_data['content_type'])
self.service = service_dict["service"]
self.category = service_dict["category"]
self.size = int(self.request.headers['Content-Length'])
body = self.payload_data['body']
self.filedata = body
self.set_status(202, "Accepted")
produce_queue.append(tracker.payload_tracker(self.request_id, self.account,
"received",
"Upload received by ingress service"))
# Offload the handling of the upload and producing to kafka
asyncio.ensure_future(
self.process_upload()
)
def read_resource(res: 'CustomResource',
ty: str,
name: str,
props: 'Inputs',
opts: 'ResourceOptions',
typ: Optional[type] = None) -> None:
if opts.id is None:
raise Exception(
"Cannot read resource whose options are lacking an ID value")
log.debug(f"reading resource: ty={ty}, name={name}, id={opts.id}")
monitor = settings.get_monitor()
# If we have type information, we'll use its and the resource's type/name metadata
# for name translations rather than using the resource's translation methods.
transform_using_type_metadata = typ is not None
# Prepare the resource, similar to a RegisterResource. Reads are deliberately similar to RegisterResource except
# that we are populating the Resource object with properties associated with an already-live resource.
#
# Same as below, we initialize the URN property on the resource, which will always be resolved.
(resolve_urn, res.__dict__["urn"]) = resource_output(res)
# Furthermore, since resources being Read must always be custom resources (enforced in the
# Resource constructor), we'll need to set up the ID field which will be populated at the end of
# the ReadResource call.
#
# Note that we technically already have the ID (opts.id), but it's more consistent with the rest
# of the model to resolve it asynchronously along with all of the other resources.
(resolve_id, res.__dict__["id"]) = resource_output(res)
# Like below, "transfer" all input properties onto unresolved futures on res.
resolvers = rpc.transfer_properties(res, props)
async def do_read():
try:
resolver = await prepare_resource(res, ty, True, False, props,
opts, typ)
# Resolve the ID that we were given. Note that we are explicitly discarding the list of
# dependencies returned to us from "serialize_property" (the second argument). This is
# because a "read" resource does not actually have any dependencies at all in the cloud
# provider sense, because a read resource already exists. We do not need to track this
# dependency.
resolved_id = await rpc.serialize_property(opts.id, [])
log.debug(f"read prepared: ty={ty}, name={name}, id={opts.id}")
# These inputs will end up in the snapshot, so if there are any additional secret
# outputs, record them here.
additional_secret_outputs = _translate_additional_secret_outputs(
res, typ, opts.additional_secret_outputs)
accept_resources = not (os.getenv(
"PULUMI_DISABLE_RESOURCE_REFERENCES", "").upper()
in {"TRUE", "1"})
req = resource_pb2.ReadResourceRequest(
type=ty,
name=name,
id=resolved_id,
parent=resolver.parent_urn,
provider=resolver.provider_ref,
properties=resolver.serialized_props,
dependencies=resolver.dependencies,
version=opts.version or "",
acceptSecrets=True,
acceptResources=accept_resources,
additionalSecretOutputs=additional_secret_outputs,
)
from ..resource import create_urn # pylint: disable=import-outside-toplevel
mock_urn = await create_urn(name, ty, resolver.parent_urn).future()
def do_rpc_call():
if monitor is None:
# If no monitor is available, we'll need to fake up a response, for testing.
return RegisterResponse(mock_urn, None,
resolver.serialized_props, None)
# If there is a monitor available, make the true RPC request to the engine.
try:
return monitor.ReadResource(req)
except grpc.RpcError as exn:
handle_grpc_error(exn)
return None
resp = await asyncio.get_event_loop().run_in_executor(
None, do_rpc_call)
except Exception as exn:
log.debug(
f"exception when preparing or executing rpc: {traceback.format_exc()}"
)
rpc.resolve_outputs_due_to_exception(resolvers, exn)
resolve_urn(None, True, False, exn)
resolve_id(None, True, False, exn)
raise
log.debug(f"resource read successful: ty={ty}, urn={resp.urn}")
resolve_urn(resp.urn, True, False, None)
resolve_id(resolved_id, True, False,
None) # Read IDs are always known.
rpc.resolve_outputs(res, resolver.serialized_props, resp.properties,
{}, resolvers, transform_using_type_metadata)
asyncio.ensure_future(RPC_MANAGER.do_rpc("read resource", do_read)())
def subscribe(self, channel, **kwargs):
sub = Subscription(self, channel, **kwargs)
self._subs[channel] = sub
asyncio.ensure_future(self._subscribe([channel]))
return sub
def invoke(tok: str,
props: 'Inputs',
opts: Optional[InvokeOptions] = None,
typ: Optional[type] = None) -> InvokeResult:
"""
invoke dynamically invokes the function, tok, which is offered by a provider plugin. The inputs
can be a bag of computed values (Ts or Awaitable[T]s), and the result is a Awaitable[Any] that
resolves when the invoke finishes.
"""
log.debug(f"Invoking function: tok={tok}")
if opts is None:
opts = InvokeOptions()
if typ and not _types.is_output_type(typ):
raise TypeError("Expected typ to be decorated with @output_type")
async def do_invoke():
# If a parent was provided, but no provider was provided, use the parent's provider if one was specified.
if opts.parent is not None and opts.provider is None:
opts.provider = opts.parent.get_provider(tok)
# Construct a provider reference from the given provider, if one was provided to us.
provider_ref = None
if opts.provider is not None:
provider_urn = await opts.provider.urn.future()
provider_id = (await opts.provider.id.future()) or rpc.UNKNOWN
provider_ref = f"{provider_urn}::{provider_id}"
log.debug(f"Invoke using provider {provider_ref}")
monitor = get_monitor()
inputs = await rpc.serialize_properties(props, {})
version = opts.version or ""
accept_resources = not (os.getenv("PULUMI_DISABLE_RESOURCE_REFERENCES",
"").upper() in {"TRUE", "1"})
log.debug(f"Invoking function prepared: tok={tok}")
req = provider_pb2.InvokeRequest(
tok=tok,
args=inputs,
provider=provider_ref,
version=version,
acceptResources=accept_resources,
)
def do_invoke():
try:
return monitor.Invoke(req), None
except grpc.RpcError as exn:
return None, grpc_error_to_exception(exn)
resp, error = await asyncio.get_event_loop().run_in_executor(
None, do_invoke)
log.debug(f"Invoking function completed: tok={tok}, error={error}")
# If the invoke failed, raise an error.
if error is not None:
return None, Exception(f"invoke of {tok} failed: {error}")
if resp.failures:
return None, Exception(
f"invoke of {tok} failed: {resp.failures[0].reason} ({resp.failures[0].property})"
)
# Otherwise, return the output properties.
ret_obj = getattr(resp, 'return')
if ret_obj:
deserialized = rpc.deserialize_properties(ret_obj)
# If typ is not None, call translate_output_properties to instantiate any output types.
return rpc.translate_output_properties(
deserialized, lambda prop: prop,
typ) if typ else deserialized, None
return None, None
async def do_rpc():
resp, exn = await RPC_MANAGER.do_rpc("invoke", do_invoke)()
# If there was an RPC level exception, we will raise it. Note that this will also crash the
# process because it will have been considered "unhandled". For semantic level errors, such
# as errors from the data source itself, we return that as part of the returned tuple instead.
if exn is not None:
raise exn
return resp
# Run the RPC callback asynchronously and then immediately await it.
# If there was a semantic error, raise it now, otherwise return the resulting value.
invoke_result, invoke_error = _sync_await(asyncio.ensure_future(do_rpc()))
if invoke_error is not None:
raise invoke_error
return InvokeResult(invoke_result)
def register_resource(res: 'Resource',
ty: str,
name: str,
custom: bool,
remote: bool,
new_dependency: Callable[[str], 'Resource'],
props: 'Inputs',
opts: Optional['ResourceOptions'],
typ: Optional[type] = None) -> None:
"""
Registers a new resource object with a given type t and name. It returns the
auto-generated URN and the ID that will resolve after the deployment has completed. All
properties will be initialized to property objects that the registration operation will resolve
at the right time (or remain unresolved for deployments).
"""
log.debug(
f"registering resource: ty={ty}, name={name}, custom={custom}, remote={remote}"
)
monitor = settings.get_monitor()
# If we have type information, we'll use its and the resource's type/name metadata
# for name translations rather than using the resource's translation methods.
transform_using_type_metadata = typ is not None
# Prepare the resource.
# Simply initialize the URN property and get prepared to resolve it later on.
# Note: a resource urn will always get a value, and thus the output property
# for it can always run .apply calls.
(resolve_urn, res.__dict__["urn"]) = resource_output(res)
# If a custom resource, make room for the ID property.
resolve_id: Optional[Callable[[Any, bool, bool, Optional[Exception]],
None]] = None
if custom:
(resolve_id, res.__dict__["id"]) = resource_output(res)
# Now "transfer" all input properties into unresolved futures on res. This way,
# this resource will look like it has all its output properties to anyone it is
# passed to. However, those futures won't actually resolve until the RPC returns
resolvers = rpc.transfer_properties(res, props)
async def do_register():
try:
resolver = await prepare_resource(res, ty, custom, remote, props,
opts, typ)
log.debug(f"resource registration prepared: ty={ty}, name={name}")
property_dependencies = {}
for key, deps in resolver.property_dependencies.items():
property_dependencies[
key] = resource_pb2.RegisterResourceRequest.PropertyDependencies(
urns=deps)
ignore_changes = _translate_ignore_changes(res, typ,
opts.ignore_changes)
additional_secret_outputs = _translate_additional_secret_outputs(
res, typ, opts.additional_secret_outputs)
# Translate the CustomTimeouts object.
custom_timeouts = None
if opts.custom_timeouts is not None:
custom_timeouts = resource_pb2.RegisterResourceRequest.CustomTimeouts(
)
# It could be an actual CustomTimeouts object.
if known_types.is_custom_timeouts(opts.custom_timeouts):
if opts.custom_timeouts.create is not None:
custom_timeouts.create = opts.custom_timeouts.create
if opts.custom_timeouts.update is not None:
custom_timeouts.update = opts.custom_timeouts.update
if opts.custom_timeouts.delete is not None:
custom_timeouts.delete = opts.custom_timeouts.delete
# Or, it could be a workaround passing in a dict.
elif isinstance(opts.custom_timeouts, dict):
if 'create' in opts.custom_timeouts:
custom_timeouts.create = opts.custom_timeouts['create']
if 'update' in opts.custom_timeouts:
custom_timeouts.update = opts.custom_timeouts['update']
if 'delete' in opts.custom_timeouts:
custom_timeouts.delete = opts.custom_timeouts['delete']
else:
raise Exception(
"Expected custom_timeouts to be a CustomTimeouts object"
)
accept_resources = not (os.getenv(
"PULUMI_DISABLE_RESOURCE_REFERENCES", "").upper()
in {"TRUE", "1"})
req = resource_pb2.RegisterResourceRequest(
type=ty,
name=name,
parent=resolver.parent_urn,
custom=custom,
object=resolver.serialized_props,
protect=opts.protect,
provider=resolver.provider_ref,
providers=resolver.provider_refs,
dependencies=resolver.dependencies,
propertyDependencies=property_dependencies,
deleteBeforeReplace=opts.delete_before_replace,
deleteBeforeReplaceDefined=opts.delete_before_replace
is not None,
ignoreChanges=ignore_changes,
version=opts.version or "",
acceptSecrets=True,
acceptResources=accept_resources,
additionalSecretOutputs=additional_secret_outputs,
importId=opts.import_,
customTimeouts=custom_timeouts,
aliases=resolver.aliases,
supportsPartialValues=True,
remote=remote,
)
from ..resource import create_urn # pylint: disable=import-outside-toplevel
mock_urn = await create_urn(name, ty, resolver.parent_urn).future()
def do_rpc_call():
if monitor is None:
# If no monitor is available, we'll need to fake up a response, for testing.
return RegisterResponse(mock_urn, None,
resolver.serialized_props, None)
# If there is a monitor available, make the true RPC request to the engine.
try:
return monitor.RegisterResource(req)
except grpc.RpcError as exn:
handle_grpc_error(exn)
return None
resp = await asyncio.get_event_loop().run_in_executor(
None, do_rpc_call)
except Exception as exn:
log.debug(
f"exception when preparing or executing rpc: {traceback.format_exc()}"
)
rpc.resolve_outputs_due_to_exception(resolvers, exn)
resolve_urn(None, True, False, exn)
if resolve_id is not None:
resolve_id(None, True, False, exn)
raise
if resp is None:
return
# At this point we would like to return successfully and call
# `rpc.resolve_outputs`, but unfortunately that itself can
# throw an exception sometimes. This was causing Pulumi
# program to hang, so the additional try..except block is used
# to propagate this exception into `rpc.resolve_outputs` which
# causes it to display.
resolve_outputs_called = False
resolve_id_called = False
resolve_urn_called = False
try:
log.debug(
f"resource registration successful: ty={ty}, urn={resp.urn}")
resolve_urn(resp.urn, True, False, None)
resolve_urn_called = True
if resolve_id is not None:
# The ID is known if (and only if) it is a non-empty string. If it's either None or an
# empty string, we should treat it as unknown. TFBridge in particular is known to send
# the empty string as an ID when doing a preview.
is_known = bool(resp.id)
resolve_id(resp.id, is_known, False, None)
resolve_id_called = True
deps = {}
rpc_deps = resp.propertyDependencies
if rpc_deps:
for k, v in rpc_deps.items():
urns = list(v.urns)
deps[k] = set(map(new_dependency, urns))
rpc.resolve_outputs(res, resolver.serialized_props, resp.object,
deps, resolvers, transform_using_type_metadata)
resolve_outputs_called = True
except Exception as exn:
log.debug(
f"exception after executing rpc: {traceback.format_exc()}")
if not resolve_outputs_called:
rpc.resolve_outputs_due_to_exception(resolvers, exn)
if not resolve_urn_called:
resolve_urn(None, True, False, exn)
if resolve_id is not None and not resolve_id_called:
resolve_id(None, True, False, exn)
raise
asyncio.ensure_future(
RPC_MANAGER.do_rpc("register resource", do_register)())
def wrapper(*args, **kwargs):
loop = asyncio.get_event_loop()
future = asyncio.ensure_future(f(*args, **kwargs))
return loop.run_until_complete(future)
def get_resource(res: 'Resource',
props: 'Inputs',
custom: bool,
urn: str,
typ: Optional[type] = None) -> None:
log.debug(f"getting resource: urn={urn}")
# If we have type information, we'll use its and the resource's type/name metadata
# for name translations rather than using the resource's translation methods.
transform_using_type_metadata = typ is not None
# Extract the resource type from the URN.
urn_parts = urn.split("::")
qualified_type = urn_parts[2]
ty = qualified_type.split("$")[-1]
# Initialize the URN property on the resource.
(resolve_urn, res.__dict__["urn"]) = resource_output(res)
# If this is a custom resource, initialize its ID property.
resolve_id: Optional[Callable[[Any, bool, bool, Optional[Exception]],
None]] = None
if custom:
(resolve_id, res.__dict__["id"]) = resource_output(res)
# Like the other resource functions, "transfer" all input properties onto unresolved futures on res.
resolvers = rpc.transfer_properties(res, props)
async def do_get():
try:
resolver = await prepare_resource(res, ty, custom, False, props,
None, typ)
monitor = settings.get_monitor()
inputs = await rpc.serialize_properties({"urn": urn}, {})
req = provider_pb2.InvokeRequest(tok="pulumi:pulumi:getResource",
args=inputs,
provider="",
version="")
def do_invoke():
try:
return monitor.Invoke(req)
except grpc.RpcError as exn:
handle_grpc_error(exn)
return None
resp = await asyncio.get_event_loop().run_in_executor(
None, do_invoke)
# If the invoke failed, raise an error.
if resp.failures:
raise Exception(
f"getResource failed: {resp.failures[0].reason} ({resp.failures[0].property})"
)
except Exception as exn:
log.debug(
f"exception when preparing or executing rpc: {traceback.format_exc()}"
)
rpc.resolve_outputs_due_to_exception(resolvers, exn)
resolve_urn(None, True, False, exn)
if resolve_id is not None:
resolve_id(None, True, False, exn)
raise
# Otherwise, grab the URN, ID, and output properties and resolve all of them.
resp = getattr(resp, 'return')
log.debug(
f"getResource completed successfully: ty={ty}, urn={resp['urn']}")
resolve_urn(resp["urn"], True, False, None)
if resolve_id:
# The ID is known if (and only if) it is a non-empty string. If it's either None or an
# empty string, we should treat it as unknown. TFBridge in particular is known to send
# the empty string as an ID when doing a preview.
is_known = bool(resp["id"])
resolve_id(resp["id"], is_known, False, None)
rpc.resolve_outputs(res, resolver.serialized_props, resp["state"], {},
resolvers, transform_using_type_metadata)
asyncio.ensure_future(RPC_MANAGER.do_rpc("get resource", do_get)())
def __callback(self, button: Gtk.Button) -> None:
if not self._user_callback:
return
task = self._user_callback(button, *self._user_args, **self._user_kwargs)
asyncio.ensure_future(task)
def call(tok: str,
props: 'Inputs',
res: Optional['Resource'] = None,
typ: Optional[type] = None) -> 'Output[Any]':
"""
call dynamically invokes the function, tok, which is offered by a provider plugin. The inputs
can be a bag of computed values (Ts or Awaitable[T]s).
"""
log.debug(f"Calling function: tok={tok}")
if typ and not _types.is_output_type(typ):
raise TypeError("Expected typ to be decorated with @output_type")
# Setup the futures for the output.
resolve_value: 'asyncio.Future' = asyncio.Future()
resolve_is_known: 'asyncio.Future[bool]' = asyncio.Future()
resolve_is_secret: 'asyncio.Future[bool]' = asyncio.Future()
resolve_deps: 'asyncio.Future[Set[Resource]]' = asyncio.Future()
from .. import Output # pylint: disable=import-outside-toplevel
out = Output(resolve_deps, resolve_value, resolve_is_known,
resolve_is_secret)
async def do_call():
try:
# Construct a provider reference from the given provider, if one is available on the resource.
provider_ref, version = None, ""
if res is not None:
if res._provider is not None:
provider_urn = await res._provider.urn.future()
provider_id = (await
res._provider.id.future()) or rpc.UNKNOWN
provider_ref = f"{provider_urn}::{provider_id}"
log.debug(f"Call using provider {provider_ref}")
version = res._version or ""
monitor = get_monitor()
# Serialize out all props to their final values. In doing so, we'll also collect all the Resources pointed to
# by any Dependency objects we encounter, adding them to 'implicit_dependencies'.
property_dependencies_resources: Dict[str, List['Resource']] = {}
inputs = await rpc.serialize_properties(
props, property_dependencies_resources)
property_dependencies = {}
for key, property_deps in property_dependencies_resources.items():
urns = set()
for dep in property_deps:
urn = await dep.urn.future()
urns.add(urn)
property_dependencies[
key] = provider_pb2.CallRequest.ArgumentDependencies(
urns=list(urns))
req = provider_pb2.CallRequest(
tok=tok,
args=inputs,
argDependencies=property_dependencies,
provider=provider_ref,
version=version,
)
def do_rpc_call():
try:
return monitor.Call(req)
except grpc.RpcError as exn:
handle_grpc_error(exn)
return None
resp = await asyncio.get_event_loop().run_in_executor(
None, do_rpc_call)
if resp is None:
return
if resp.failures:
raise Exception(
f"call of {tok} failed: {resp.failures[0].reason} ({resp.failures[0].property})"
)
log.debug(f"Call successful: tok={tok}")
value = None
is_secret = False
deps: Set['Resource'] = set()
ret_obj = getattr(resp, "return")
if ret_obj:
deserialized = rpc.deserialize_properties(ret_obj)
# Keep track of whether we need to mark the resulting output a secret,
# and unwrap each individual value.
for k, v in deserialized.items():
if rpc.is_rpc_secret(v):
is_secret = True
deserialized[k] = rpc.unwrap_rpc_secret(v)
# Combine the individual dependencies into a single set of dependency resources.
rpc_deps = resp.returnDependencies
deps_urns: Set[str] = {
urn
for v in rpc_deps.values() for urn in v.urns
} if rpc_deps else set()
from ..resource import DependencyResource # pylint: disable=import-outside-toplevel
deps = set(map(DependencyResource, deps_urns))
# If typ is not None, call translate_output_properties to instantiate any output types.
value = rpc.translate_output_properties(
deserialized, lambda prop: prop,
typ) if typ else deserialized
resolve_value.set_result(value)
resolve_is_known.set_result(True)
resolve_is_secret.set_result(is_secret)
resolve_deps.set_result(deps)
except Exception as exn:
log.debug(
f"exception when preparing or executing rpc: {traceback.format_exc()}"
)
resolve_value.set_exception(exn)
resolve_is_known.set_exception(exn)
resolve_is_secret.set_exception(exn)
resolve_deps.set_result(set())
raise
asyncio.ensure_future(RPC_MANAGER.do_rpc("call", do_call)())
return out
