def test_heartbeat_current_op_over_limit(self): self.patch_cfg('pyon.ion.process.CFG', {'container':{'timeout':{'heartbeat_proc_count_threshold':2}}}) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target def fake_op(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() self.addCleanup(listeninev.set) # allow graceful termination self.addCleanup(p.stop) ar = p._routing_call(fake_op, None, listenoutev, listeninev) listenoutev.wait(timeout=5) # wait for ctrl thread to run our op # make sure it's over the threshold for x in xrange(3): hb = p.heartbeat() self.assertEquals((True, True, False), hb)
def test_heartbeat_listener_dead(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) listeninev.set() # stop the listen loop p.thread_manager.children[1].join(timeout=5) # wait for listen loop to terminate hb = p.heartbeat() self.assertEquals((False, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def test_heartbeat_current_op_over_limit(self): self.patch_cfg( 'pyon.ion.process.CFG', {'cc': { 'timeout': { 'heartbeat_proc_count_threshold': 2 } }}) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target def fake_op(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() self.addCleanup(listeninev.set) # allow graceful termination self.addCleanup(p.stop) ar = p._routing_call(fake_op, None, listenoutev, listeninev) listenoutev.wait(timeout=5) # wait for ctrl thread to run our op # make sure it's over the threshold for x in xrange(3): hb = p.heartbeat() self.assertEquals((True, True, False), hb)
def test_heartbeat_with_current_op_multiple_times(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target def fake_op(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() self.addCleanup(listeninev.set) # allow graceful termination self.addCleanup(p.stop) ar = p._routing_call(fake_op, None, listenoutev, listeninev) listenoutev.wait(timeout=5) # wait for ctrl thread to run our op for x in xrange(5): hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(5, p._heartbeat_count) self.assertEquals(ar, p._heartbeat_op)
def mediated_transfer(initiator_app, target_app, asset, amount, identifier=None): """ Nice to read shortcut to make a MediatedTransfer. The secret will be revealed and the apps will be synchronized. """ # pylint: disable=too-many-arguments assetmanager = initiator_app.raiden.managers_by_asset_address[asset] has_channel = initiator_app.raiden.address in assetmanager.partneraddress_channel # api.transfer() would do a DirectTransfer if has_channel: transfermanager = assetmanager.transfermanager # Explicitly call the default identifier creation since this mock # function here completely skips the `transfer_async()` call. if not identifier: identifier = transfermanager.create_default_identifier(target_app.raiden.address) result = AsyncResult() task = StartMediatedTransferTask( transfermanager, amount, identifier, target_app.raiden.address, result, ) task.start() result.wait() else: initiator_app.raiden.api.transfer( asset, amount, target_app.raiden.address, identifier )
def mediated_transfer(initiator_app, target_app, token, amount, identifier=None): """ Nice to read shortcut to make a MediatedTransfer. The secret will be revealed and the apps will be synchronized. """ # pylint: disable=too-many-arguments tokenmanager = initiator_app.raiden.managers_by_token_address[token] has_channel = initiator_app.raiden.address in tokenmanager.partneraddress_channel # api.transfer() would do a DirectTransfer if has_channel: transfermanager = tokenmanager.transfermanager # Explicitly call the default identifier creation since this mock # function here completely skips the `transfer_async()` call. if not identifier: identifier = transfermanager.create_default_identifier(target_app.raiden.address) result = AsyncResult() task = StartMediatedTransferTask( transfermanager, amount, identifier, target_app.raiden.address, result, ) task.start() result.wait() else: initiator_app.raiden.api.transfer( token, amount, target_app.raiden.address, identifier )
def test_heartbeat_with_listeners(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() mocklistener.listen = lambda *a, **kw: fake_listen(listenoutev, listeninev) p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) # now test heartbeat! hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def send_request(self, host, request_id, entity_type, entity_id, method, *args, **kwargs): self.connect() request = RpcRequest() if host is not None: request.host = host else: request.host = get_server_unique_id() if request_id is not None: request.request_id = request_id request.entity_type = entity_type request.entity_id = entity_id request.method = method request.args = args request.kwargs = kwargs self._conn.send_message(request) future = AsyncResult() add_future(request.request_id, future) future.wait(5) resp: RpcResponse = future.get() return resp.response
def _event(): # 协程通信 e = Event() ae = AsyncResult() e.wait() # 唤醒 e.set() # 唤醒 ae.wait() # 唤醒 ae.set('') # 唤醒,可传递信息
def aimbot(): dw = dolphinWatch.DolphinConnection() connected_event = AsyncResult() dw.onConnect(connected_event.set) dw.connect() connected_event.wait() while True: time.sleep(0.1) dw.write32(pointer + 0x6405e0, 1) dw.save("G:/bla.sav")
def test_foo(self): router = QueueRouter() resultSlot1 = AsyncResult() resultSlot2 = AsyncResult() gevent.spawn(subscriber1, router, resultSlot1) gevent.spawn(subscriber2, router, resultSlot2) gevent.sleep(0) # yield to subscribers for setup gevent.spawn(publisher, router) result1 = resultSlot1.wait() result2 = resultSlot2.wait() expected = [('foobar', 0), ('foobar', 1), ('foobar', 2)] self.assertEqual(result1, expected) self.assertEqual(result2, expected)
class LogListenerTask(Task): def __init__(self, filter_, callback, contract_translator): super(LogListenerTask, self).__init__() self.filter_ = filter_ self.callback = callback self.contract_translator = contract_translator self.stop_event = AsyncResult() self.sleep_time = 0.5 def _run(self): # pylint: disable=method-hidden stop = None while stop is None: filter_changes = self.filter_.changes() for log_event in filter_changes: event = self.contract_translator.decode_event( log_event['topics'], log_event['data'], ) if event is not None: originating_contract = log_event['address'] self.callback(originating_contract, event) stop = self.stop_event.wait(self.sleep_time) def stop(self): self.stop_event.set(True)
class ForwardSecretTask(Task): timeout = TransferTask.timeout_per_hop def __init__(self, transfermanager, hashlock, recipient): self.transfermanager = transfermanager self.recipient = recipient self.hashlock = hashlock self.raiden = transfermanager.assetmanager.raiden super(ForwardSecretTask, self).__init__() log.info("INIT", task=self) self.transfermanager.on_task_started(self) def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address)) def _run(self): self.event = AsyncResult() # http://www.gevent.org/gevent.event.html timeout = self.timeout msg = self.event.wait(timeout) if not msg: log.error("TIMEOUT! " * 5) # TransferTimeout is of no use, SecretRequest was for sender return self.on_completion(False) assert isinstance(msg, Secret) assert msg.hashlock == self.hashlock fwd = Secret(msg.secret) self.raiden.sign(fwd) self.raiden.send(self.recipient, fwd) return self.on_completion(True)
class ForwardSecretTask(gevent.Greenlet): timeout = TransferTask.timeout_per_hop def __init__(self, transfermanager, hashlock, recipient): self.recipient = recipient self.hashlock = hashlock self.raiden = transfermanager.assetmanager.raiden super(ForwardSecretTask, self).__init__() print "INIT", self def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address)) def on_event(self, msg): print "SET EVENT {} {} {}".format(self, id(self.event), msg) if self.event.ready(): print "ALREADY HAD EVENT {} {} now {}".format(self, self.event.get(), msg) assert self.event and not self.event.ready() self.event.set(msg) def _run(self): self.event = AsyncResult() # http://www.gevent.org/gevent.event.html timeout = self.timeout msg = self.event.wait(timeout) if not msg: print "TIMEOUT! " * 5 # TransferTimeout is of no use, SecretRequest was for sender return False assert isinstance(msg, Secret) assert msg.hashlock == self.hashlock fwd = Secret(msg.secret) self.raiden.sign(fwd) self.raiden.send(self.recipient, fwd)
def asd(): dw = dolphinWatch.DolphinConnection() connected_event = AsyncResult() dw.onConnect(connected_event.set) dw.connect() connected_event.wait() pointer = AsyncResult() dw.read32(0x918F4FFC, pointer.set) pointer = pointer.get() print("Pointer: %x" % pointer) #0x91951494 # phrase 0x5A9A4 # trainer name 0x5A97c dw.writeMulti(pointer + 0x5A97c, [0x00, 0x4f, 0x00, 0x4e, 0x00, 0x45, 0x00, 0x48, 0x00, 0x41, 0x00, 0x4e, 0x00, 0x44, 0x00, 0x00])
def test_killing_unlinked(self): e = AsyncResult() def func(): try: raise ExpectedError('test_killing_unlinked') except: e.set_exception(sys.exc_info()[1]) p = gevent.spawn_link(func) try: try: e.wait() except ExpectedError: pass finally: p.unlink() # this disables LinkedCompleted that otherwise would be raised by the next line sleep(DELAY)
class TestAsyncResult(object): def __init__(self): self.event = AsyncResult() def run(self): producers = [gevent.spawn(self._producer, i) for i in xrange(3)] consumers = [gevent.spawn(self._consumer, i) for i in xrange(3)] tasks = [] tasks.extend(producers) tasks.extend(consumers) gevent.joinall(tasks) def _producer(self, pid): print("I'm producer %d and now I don't want consume to do something" % (pid,)) sleeptime = random.randint(5, 10) * 0.01 print("Sleeping time is %f" % (sleeptime, )) gevent.sleep(sleeptime) print("I'm producer %d and now consumer could do something." % (pid,)) self.event.set('producer pid %d' % (pid, )) def _consumer(self, pid): print("I'm consumer %d and now I'm waiting for producer" % (pid,)) gevent.sleep(random.randint(0, 5) * 0.01) value = self.event.wait() print("I'm consumer %d. Value is %r and now I can do something" % (pid, value))
class ForwardSecretTask(Task): def __init__(self, transfermanager, hashlock, recipient, msg_timeout): self.transfermanager = transfermanager self.recipient = recipient self.hashlock = hashlock self.msg_timeout = msg_timeout self.raiden = transfermanager.raiden super(ForwardSecretTask, self).__init__() log.info('INIT', task=self) self.transfermanager.on_task_started(self) def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address)) def _run(self): # pylint: disable=method-hidden self.event = AsyncResult() # http://www.gevent.org/gevent.event.html msg = self.event.wait(self.msg_timeout) # returns None if msg_timeout is reached and event-value wasn't set() if not msg: log.error('TIMEOUT! ' * 5) # TransferTimeout is of no use, SecretRequest was for sender return self.on_completion(False) assert isinstance(msg, Secret) assert msg.hashlock == self.hashlock fwd = Secret(msg.secret) self.raiden.sign(fwd) self.raiden.send(self.recipient, fwd) return self.on_completion(True)
class ForwardSecretTask(Task): timeout = TIMEOUT_PER_HOP def __init__(self, transfermanager, hashlock, recipient): self.transfermanager = transfermanager self.recipient = recipient self.hashlock = hashlock self.raiden = transfermanager.raiden super(ForwardSecretTask, self).__init__() log.info('INIT', task=self) self.transfermanager.on_task_started(self) def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address)) def _run(self): # pylint: disable=method-hidden self.event = AsyncResult() # http://www.gevent.org/gevent.event.html timeout = self.timeout msg = self.event.wait(timeout) if not msg: log.error('TIMEOUT! ' * 5) # TransferTimeout is of no use, SecretRequest was for sender return self.on_completion(False) assert isinstance(msg, Secret) assert msg.hashlock == self.hashlock fwd = Secret(msg.secret) self.raiden.sign(fwd) self.raiden.send(self.recipient, fwd) return self.on_completion(True)
def wait(self, *args, **kwargs): result = AsyncResult() match = args[-1] def _f(e): if match(e): result.set(e) return result.wait(kwargs.pop('timeout', None))
def test_killing_unlinked(self): e = AsyncResult() def func(): try: raise ExpectedError('test_killing_unlinked') except: e.set_exception(sys.exc_info()[1]) p = gevent.spawn_link(func) try: try: e.wait() except ExpectedError: pass finally: p.unlink( ) # this disables LinkedCompleted that otherwise would be raised by the next line sleep(DELAY)
def once(self, *args, **kwargs): result = AsyncResult() li = None def _f(e): result.set(e) li.detach() li = self.on(*args + (_f, )) return result.wait(kwargs.pop('timeout', None))
def _read_response(self, index, timeout): rs = AsyncResult() self._resps[index] = rs resp = rs.wait(timeout) self._resps.pop(index, None) if not rs.successful(): error = rs.exception if error is None: error = Timeout raise error return resp
class EndMediatedTransferTask(Task): """ Task that request a secret for a registered transfer. """ def __init__(self, transfermanager, originating_transfer): super(EndMediatedTransferTask, self).__init__() self.address = transfermanager.assetmanager.raiden.address self.transfermanager = transfermanager self.originating_transfer = originating_transfer hashlock = originating_transfer.lock.hashlock self.transfermanager.register_task_for_hashlock(self, hashlock) def __repr__(self): return '<{} {}>'.format( self.__class__.__name__, pex(self.address), ) def _run(self): # pylint: disable=method-hidden transfer = self.originating_transfer assetmanager = self.transfermanager.assetmanager raiden = assetmanager.raiden transfer_details = '{} -> {} hash:{}'.format( pex(transfer.target), pex(transfer.initiator), pex(transfer.hash), ) log.debug('END MEDIATED TRANSFER {}'.format(transfer_details)) secret_request = SecretRequest(transfer.lock.hashlock) raiden.sign(secret_request) raiden.send(transfer.initiator, secret_request) self.event = AsyncResult() response = self.event.wait(raiden.config['msg_timeout']) if response is None: log.error('SECRETREQUEST TIMED OUT!') self.transfermanager.on_hashlock_result(transfer.hashlock, False) return if not isinstance(response, Secret): raise Exception('Invalid message received.') if sha3(response.secret) != transfer.lock.hashlock: raise Exception('Invalid secret received.') # update all channels and propagate the secret assetmanager.register_secret(response.secret) self.transfermanager.on_hashlock_result(transfer.lock.hashlock, True)
def call(self, fname, data, noresult=False, timeout=None): """ server -> client: rpc call """ gw = self.gw_mgr.rpc_gws[self.gwid] data = ujson.dumps({FIELD_FUNC: fname, FIELD_STATUS: 1, FIELD_DATA: data, FIELD_ERROR: ''}) mid = next(self._call_id) gw.send(self.pid, self.route_id, b'%s%s' % (struct.pack(DATA_HEAD_STRUCT, TAG_REQ, mid), bytes(data, 'utf-8'))) if noresult: return result = AsyncResult() self._waits[mid] = result return result.wait(timeout)
class LogListenerTask(Task): def __init__(self, listener_name, filter_, callback, contract_translator, events_poll_timeout=DEFAULT_EVENTS_POLL_TIMEOUT): super(LogListenerTask, self).__init__() self.listener_name = listener_name self.filter_ = filter_ self.callback = callback self.contract_translator = contract_translator self.stop_event = AsyncResult() self.sleep_time = events_poll_timeout # exposes the AsyncResult timer, this allows us to raise the timeout # inside this Task to force an update: # # task.kill(task.timeout) # self.timeout = None def __repr__(self): return '<LogListenerTask {}>'.format(self.listener_name) def _run(self): # pylint: disable=method-hidden stop = None while stop is None: filter_changes = self.filter_.changes() for log_event in filter_changes: log.debug('New Events', task=self.listener_name) event = self.contract_translator.decode_event( log_event['topics'], log_event['data'], ) if event is not None: originating_contract = log_event['address'] try: self.callback(originating_contract, event) except: log.exception('unexpected exception on log listener') self.timeout = Timeout(self.sleep_time) # wait() will call cancel() stop = self.stop_event.wait(self.timeout) def stop(self): self.stop_event.set(True)
def test_heartbeat_with_listeners(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() mocklistener.listen = lambda *a, **kw: fake_listen( listenoutev, listeninev) p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup( listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) # now test heartbeat! hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def test_heartbeat_listener_dead(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup( listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) listeninev.set() # stop the listen loop p.thread_manager.children[1].join( timeout=5) # wait for listen loop to terminate hb = p.heartbeat() self.assertEquals((False, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
class RunnableTest(Runnable): def __init__(self): super().__init__() def start(self): self._stop_event = AsyncResult() super().start() def _run(self, *args: Any, **kwargs: Any) -> None: while self._stop_event and self._stop_event.wait(0.5) is not True: gevent.sleep(0.1) return def stop(self): if self._stop_event: self._stop_event.set(True)
class BaseConnection(object): """A connection to an AMQP server. This class deals with establishing and reconnecting errors, and routes messages received off the wire to handlers registered with the corresponding channel. It also handles sending and receiving heartbeat frames to detect when the connection has been lost. AMQP sends connection-level errors (connection.close) that cause the connection to close; to support this we dispatch such errors to all channels. """ frame_max = 131072 # adjusted by Tune frame channel_max = 65535 # adjusted by Tune method MAX_SEND_QUEUE = 32 # frames server_properties = None # properties and capabilities sent by the remote connection heartbeat = 0 # seconds between heartbeats. 0 means off. # NB. heartbeats don't start until they are negotiated def __init__(self, amqp_url='amqp:///', heartbeat=30, debug=False): super(BaseConnection, self).__init__() self.channel_id = 0 self.channels = {} self.connect_lock = RLock() self.channels_lock = RLock() self.queue = None self.state = STATE_DISCONNECTED self.disconnect_event = Event() self.debug = debug # Negotiate for heartbeats self.requested_heartbeat = heartbeat (self.username, self.password, self.vhost, self.host, self.port) = \ parse_amqp_url(str(amqp_url)) def allocate_channel(self): """Create a new channel.""" self.connect() # This is a no-op if we're already connected with self.channels_lock: for i in xrange(self.channel_max): self.channel_id = self.channel_id % (self.channel_max - 1) + 1 if self.channel_id not in self.channels: break else: raise ChannelError("No available channels!") id = self.channel_id chan = MessageChannel(self, id) self.channels[id] = chan chan.channel_open() return chan def _remove_channel(self, id): """Remove a channel (presumably because it has closed.)""" with self.channels_lock: del(self.channels[id]) def connected(self): return self.state == STATE_CONNECTED def connect(self): """Open the connection to the server. This method blocks until the connection has been opened and handshaking is complete. If connection fails, an exception will be raised instead. """ # Don't bother acquiring the lock if we are already connected if self.connected(): return with self.connect_lock: # Check again now we've acquired the lock, as the previous holder # of the lock would typically have completed the connection. if self.connected(): return self.disconnect_event.clear() self.connected_event = AsyncResult() self._connect() v = self.connected_event.wait() # Block until the connection is # properly ready # FFS, AsyncResult.set_exception doesn't work in gevent 1.0b4 # so we just use normal setting, but with exception types if isinstance(v, Exception): raise v def _connect(self): """Connect to the remote server and start reader/writer greenlets.""" # NB. this method isn't safe to call directly; must only be called # via connect() which adds the necessary locking self.state = STATE_CONNECTING try: try: addrinfo = socket.getaddrinfo(self.host, self.port, socket.AF_INET6, socket.SOCK_STREAM) except socket.gaierror: addrinfo = socket.getaddrinfo(self.host, self.port, socket.AF_INET, socket.SOCK_STREAM) (family, socktype, proto, canonname, sockaddr) = addrinfo[0] self.sock = socket.socket(family, socktype, proto) self.sock.connect(sockaddr) except: self.state = STATE_DISCONNECTED raise # Set up channel state self.channel_id = 1 self.channels = {} # Set up connection greenlets self.queue = Queue(self.MAX_SEND_QUEUE) reader = gevent.spawn(self.do_read, self.sock) writer = gevent.spawn(self.do_write, self.sock, self.queue) reader.link(lambda reader: writer.kill()) writer.link(lambda writer: reader.kill()) self.reader = reader self.writer = writer def _on_error(self, exc): """Dispatch a connection error to all channels.""" for id, channel in self.channels.items(): channel.on_error(exc) self.fire_async('error') def _on_connect(self): """Called when the connection is fully open.""" self.state = STATE_CONNECTED self.connected_event.set("Connected!") self.fire_async('connect') def _on_abnormal_disconnect(self, exc): """Called when the connection has been abnormally disconnected.""" self.state = STATE_DISCONNECTED try: self._on_error(exc) except Exception: pass while True: try: self.connect() except Exception: gevent.sleep(5) else: break def _on_normal_disconnect(self): """Called when the connection has closed.""" self.state = STATE_DISCONNECTED self._on_error(ConnectionError("Connection closed.")) self.queue.put(None) self.disconnect_event.set() def do_read(self, sock): """Run a reader greenlet. This method will read a preamble then loop forever reading frames off the wire and dispatch them to channels. """ try: reader = BufferedReader(sock) TIMEOUT_EXC = ConnectionError('Heartbeat timeout') while True: if self.heartbeat: t = gevent.Timeout( seconds=self.heartbeat * 2, exception=TIMEOUT_EXC) t.start() else: t = None frame_header = reader.read(FRAME_HEADER.size) frame_type, channel, size = \ FRAME_HEADER.unpack(frame_header) payload = reader.read(size + 1) if t is not None: t.cancel() if payload[-1] != '\xCE': raise ConnectionError('Received invalid frame data') if self.debug: self._debug_print('s->c', frame_header + payload) buffer = DecodeBuffer(payload) if frame_type == 0x01: # Method frame method_id, = buffer.read('!I') frame = spec.METHODS[method_id].decode(buffer) self.inbound_method(channel, frame) elif frame_type == 0x02: # header frame class_id, body_size = buffer.read('!HxxQ') props = spec.PROPS[class_id](buffer) self.inbound_props(channel, body_size, props) elif frame_type == 0x03: # body frame self.inbound_body(channel, payload[:-1]) elif frame_type in [0x04, 0x08]: # Heartbeat frame # # Catch it as both 0x04 and 0x08 - see # http://www.rabbitmq.com/amqp-0-9-1-errata.html#section_29 # # We don't need to handle this specifically - it's already # covered by the read timeout above. pass else: raise ConnectionError("Unknown frame type") except (gevent.GreenletExit, Exception) as e: self.connected_event.set(e) if self.state in [STATE_CONNECTED, STATE_CONNECTING]: # Spawn a new greenlet to run the reconnect loop gevent.spawn(self._on_abnormal_disconnect, e) else: self.state = STATE_DISCONNECTED def inbound_method(self, channel, frame): """Dispatch an inbound method.""" try: c = self.channels[channel] except KeyError: if frame.name == 'connection.start': c = StartChannel(self, channel) self.channels[channel] = c else: return c._on_method(frame) def inbound_props(self, channel, body_size, props): """Dispatch an inbound properties frame.""" try: c = self.channels[channel] except KeyError: return c._on_headers(body_size, props) def inbound_body(self, channel, payload): """Dispatch an inbound body frame.""" try: c = self.channels[channel] except KeyError: return c._on_body(payload) def do_write(self, sock, queue): """Run a writer greenlet. This greenlet will loop until the connection closes, writing frames from the queue. """ # Write the protocol header sock.sendall(spec.PREAMBLE) # Enter a send loop while True: if self.heartbeat: try: msg = queue.get(timeout=self.heartbeat) except Empty: # Send heartbeat sock.sendall( FRAME_HEADER.pack(0x08, 0, 0) + '\xCE' ) continue else: msg = queue.get() if msg is None: return if self.debug: self._debug_print('s<-c', msg) sock.sendall(msg) def _debug_print(self, direction, msg): """Decode a frame and print it for debugging.""" try: # Print method, for debugging type, channel, size = FRAME_HEADER.unpack_from(msg) if type == 1: method_id = struct.unpack_from('!I', msg, FRAME_HEADER.size)[0] print direction, spec.METHODS[method_id].name else: print direction, { 2: '[headers %d bytes]', 3: '[payload %d bytes]', 4: '[heartbeat %d bytes]', }[type] % size except Exception: import traceback traceback.print_exc() def _send_frames(self, channel, frames): """Send a sequence of frames on channel. Each frame will be put onto a queue for the writer to write, which could cause the calling greenlet to block if the queue is full. This should cause large outgoing messages to be spliced together so that no caller is starved of service while a large message is sending. """ assert channel in self.channels for type, payload in frames: fdata = ''.join([ FRAME_HEADER.pack(type, channel, len(payload)), payload, '\xCE' ]) self.queue.put(fdata) def _tune(self, frame_max, channel_max, heartbeat=0): """Adjust connection parameters. Called in response to negotiation with the server. """ frame_max = frame_max if frame_max != 0 else 2**19 # limit the maximum frame size, to ensure messages are multiplexed self.frame_max = min(131072, frame_max) self.channel_max = channel_max if channel_max > 0 else 65535 if heartbeat: self.heartbeat = min(self.requested_heartbeat, heartbeat) else: self.heartbeat = self.requested_heartbeat def close(self, block=True, timeout=2): if self.state in [STATE_CONNECTED, STATE_CONNECTING]: self.state = STATE_DISCONNECTING if block: self.channels[0].connection_close() self.writer.join(timeout=timeout) else: self.channels[0]._send( spec.FrameConnectionClose(200, '', 0, 0) ) def set_dispatcher(self, dispatcher): """Set the event dispatcher. BaseConnection holds a weak ref to its dispatcher so that our connection threads don't keep it alive. """ self._dispatcher = weakref.ref(dispatcher) def get_dispatcher(self): """Get the event dispatcher or return None if there is no dispatcher.""" if self._dispatcher: return self._dispatcher() dispatcher = property(get_dispatcher, set_dispatcher) def fire(self, event, *args, **kwargs): """Fire an event using the dispatcher.""" d = self.dispatcher if d: return self.dispatcher.fire(event, *args, **kwargs) def fire_async(self, event, *args, **kwargs): """Asynchronously fire an event using the dispatcher.""" d = self.dispatcher if d: return self.dispatcher.fire_async(event, *args, **kwargs)
class MediatedTransferTask(Task): # Normal Operation (Transfer A > C) # A: Initiator Creates Secret # A: MediatedTransfer > B # B: MediatedTransfer > C # C: SecretRequest > A (implicitly signs, that valid transfer was received) # A: Secret > C # C: Secret > B # Timeout (Transfer A > C) # A: Initiator Creates Secret # A: MediatedTransfer > B # B: MediatedTransfer > C # Failure: No Ack from C # B: TransferTimeout > A # Resolution: A won't reveal the secret, tries new transfer, B bans C # CancelTransfer (Transfer A > D) # A: Initiator Creates Secret # A: MediatedTransfer > B # B: MediatedTransfer > C # Failure: C can not establish path to D (e.g. insufficient distributable, no active node) # C: CancelTransfer > B (levels out balance) # B: MediatedTransfer > C2 # C2: MediatedTransfer > D def __init__(self, transfermanager, amount, target, hashlock, expiration, originating_transfer=None, secret=None): # fee! import transfermanager as transfermanagermodule assert isinstance(transfermanager, transfermanagermodule.TransferManager) self.amount = amount self.assetmanager = transfermanager.assetmanager self.event = None self.fee = 0 # FIXME: calculate fee self.hashlock = hashlock self.expiration = expiration self.originating_transfer = originating_transfer self.raiden = transfermanager.raiden self.secret = secret self.target = target self.transfermanager = transfermanager self.isinitiator = bool(secret) if originating_transfer and secret: raise ValueError('Cannot set both secret and originating_transfer') if not (originating_transfer or secret): raise ValueError('Either originating_transfer or secret needs to be informed') if originating_transfer and not isinstance(originating_transfer, LockedTransfer): raise ValueError('originating_transfer needs to be a LockedTransfer') if self.isinitiator: self.initiator = self.raiden.address else: self.initiator = originating_transfer.initiator super(MediatedTransferTask, self).__init__() self.transfermanager.on_task_started(self) def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address)) def _run(self): # pylint: disable=method-hidden for path, channel in self.get_best_routes(): next_hop = path[1] mediated_transfer = channel.create_mediatedtransfer( self.initiator, self.target, self.fee, self.amount, # HOTFIX: you cannot know channel.settle_timeout beforehand,since path is not yet defined # FIXME: implement expiration adjustment in different task (e.g. 'InitMediatedTransferTask') self.expiration if self.expiration is not None else channel.settle_timeout - 1, self.hashlock, ) self.raiden.sign(mediated_transfer) channel.register_transfer(mediated_transfer) log.debug('MEDIATED TRANSFER initiator={} {}'.format( pex(mediated_transfer.initiator), lpex(path), )) msg_timeout = self.raiden.config['msg_timeout'] # timeout not dependent on expiration (canceltransfer/transfertimeout msgs), # but should be set shorter than the expiration msg = self.send_transfer_and_wait(next_hop, mediated_transfer, path, msg_timeout) log.debug('MEDIATED TRANSFER RETURNED {} {}'.format( pex(self.raiden.address), msg, )) result = self.check_path(msg, channel) # `next_hop` didn't get a response from any of it's, let's try the # next path if isinstance(result, TransferTimeout): continue # `next_hop` doesn't have any path with a valid channel to proceed, # try the next path if isinstance(result, CancelTransfer): continue # `check_path` failed, try next path if result is None: continue return self.on_completion(result) # No suitable path avaiable (e.g. insufficient distributable, no active node) # Send CancelTransfer to the originating node, this has the effect of # backtracking in the graph search of the raiden network. if self.originating_transfer: from_address = self.originating_transfer.sender from_transfer = self.originating_transfer from_channel = self.assetmanager.channels[from_address] log.debug('CANCEL MEDIATED TRANSFER from={} {}'.format( pex(from_address), pex(self.raiden.address), )) cancel_transfer = from_channel.create_canceltransfer_for(from_transfer) self.raiden.sign(cancel_transfer) from_channel.register_transfer(cancel_transfer) self.raiden.send(from_address, cancel_transfer) else: log.error('UNABLE TO COMPLETE MEDIATED TRANSFER target={} amount={}'.format( pex(self.target), self.amount, )) return self.on_completion(False) def get_best_routes(self): """ Yield a two-tuple (path, channel) that can be used to mediate the transfer. The result is ordered from the best to worst path. """ available_paths = self.assetmanager.channelgraph.get_shortest_paths( self.raiden.address, self.target, ) for path in available_paths: assert path[0] == self.raiden.address assert path[1] in self.assetmanager.channels assert path[-1] == self.target partner = path[1] channel = self.assetmanager.channels[partner] if not channel.isopen: continue # we can't intermediate the transfer if we don't have enough funds if self.amount > channel.distributable: continue # Our partner won't accept a locked transfer that can expire after # the settlement period, otherwise the secret could be revealed # after channel is settled and he would lose the asset, or before # the minimum required. # FIXME Hotfix, see self._run() if not self.expiration: expiration = channel.settle_timeout - 1 else: expiration = self.expiration if not (self.raiden.chain.block_number + channel.reveal_timeout <= expiration < channel.settle_timeout): log.debug( 'expiration is too large, channel/path cannot be used', expiration=expiration, channel_locktime=channel.settle_timeout, nodeid=pex(path[0]), partner=pex(path[1]), ) continue yield (path, channel) def check_path(self, msg, channel): if isinstance(msg, CancelTransfer): return None # try with next path elif isinstance(msg, TransferTimeout): # stale hashlock if not self.isinitiator: self.raiden.send(self.originating_transfer.sender, msg) return False elif isinstance(msg, Secret): assert self.originating_transfer assert msg.hashlock == self.hashlock if self.originating_transfer.sender != self.originating_transfer.initiator: fwd = Secret(msg.secret) self.raiden.sign(fwd) self.raiden.send(self.originating_transfer.sender, fwd) else: log.warning('NOT FORWARDING SECRET TO ININTIATOR') return True elif isinstance(msg, SecretRequest): assert self.isinitiator # lock.target can easilly be tampered, ensure that we are receiving # the SecretRequest from the correct node if msg.sender != self.target: log.error('Tampered SecretRequest', secret_request=msg) return None # try the next available path # TODO: the lock.amount can easily be tampered, check the `target` # locked transfer has the correct `amount` msg = Secret(self.secret) self.raiden.sign(msg) self.raiden.send(self.target, msg) # TODO: Guarantee that `target` received the secret, otherwise we # updated the channel and the first hop will receive the asset, but # none of the other channels will make the transfer channel.claim_locked(self.secret) return True return None def send_transfer_and_wait(self, recipient, transfer, path, msg_timeout): """ Send `transfer` to `recipient` and wait for the response. Args: recipient (address): The address of the node that will receive the message. transfer: The transfer message. path: The current path that is being tried. msg_timeout: How long should we wait for a response from `recipient`. Returns: TransferTimeout: If the other end didn't respond """ self.event = AsyncResult() self.raiden.send(recipient, transfer) # The event is set either when a relevant message is received or we # reach the timeout. # # The relevant messages are: CancelTransfer, TransferTimeout, # SecretRequest, or Secret msg = self.event.wait(msg_timeout) # Timed out if msg is None: log.error('TIMEOUT [{}]! recipient={} didnt respond path={}'.format( msg_timeout, pex(recipient), lpex(path), )) transfer_timeout = TransferTimeout( echo=transfer.hash, hashlock=transfer.lock.hashlock, ) self.raiden.sign(transfer_timeout) return transfer_timeout log.debug( 'HAVE EVENT {} {}'.format(self, msg), node=pex(self.raiden.address), ) if isinstance(msg, CancelTransfer): assert msg.lock.hashlock == transfer.lock.hashlock assert msg.lock.amount == transfer.lock.amount assert msg.recipient == transfer.sender == self.raiden.address channel = self.assetmanager.channels[msg.sender] channel.register_transfer(msg) return msg elif isinstance(msg, TransferTimeout): assert msg.echo == transfer.hash return msg # send back StaleHashLock, we need new hashlock elif isinstance(msg, Secret): # done exit assert msg.hashlock == self.hashlock # channel = self.assetmanager.channels[msg.recipient] # channel.claim_locked(msg.secret) # fixme this is also done by assetmanager return msg elif isinstance(msg, SecretRequest): # reveal secret log.info('SECRETREQUEST RECEIVED {}'.format(msg)) assert msg.sender == self.target return msg raise NotImplementedError()
class Popen(SubprocessProtocol): def __init__(self, args, bufsize=-1, stdin=None, stdout=None, stderr=None, shell=False, universal_newlines=False, **kwargs): """Create new Popen instance.""" assert not universal_newlines, "universal_newlines must be False" hub = get_hub() self.pid = None self.returncode = None self.universal_newlines = universal_newlines self.result = AsyncResult() self.stdin = None self.stdout = None self.stderr = None self._transport = None if shell: hub.wait_async(hub.loop.subprocess_shell( lambda: _DelegateProtocol(self), *args, stdin=stdin, stdout=stdout, stderr=stderr, bufsize=bufsize, **kwargs)) else: hub.wait_async(hub.loop.subprocess_exec( lambda: _DelegateProtocol(self), *args, stdin=stdin, stdout=stdout, stderr=stderr, bufsize=bufsize, **kwargs)) def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode) def communicate(self, input=None, timeout=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional input argument should be a string to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr).""" #TODO: timeout greenlets = [] if self.stdin: greenlets.append(spawn(write_and_close, self.stdin, input)) if self.stdout: stdout = spawn(self.stdout.read) greenlets.append(stdout) else: stdout = None if self.stderr: stderr = spawn(self.stderr.read) greenlets.append(stderr) else: stderr = None joinall(greenlets) if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() self.wait() return (None if stdout is None else stdout.value or '', None if stderr is None else stderr.value or '') def poll(self): return self._internal_poll() def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) # XXX unlink def send_signal(self, sig): self._transport.send_signal(sig) def terminate(self): self._transport.terminate() def kill(self): #noinspection PyProtectedMember if self._transport._proc is not None: self._transport.kill() if mswindows: # # Windows methods # def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: self.returncode = GetExitCodeProcess(self._handle) self.result.set(self.returncode) return self.returncode def rawlink(self, callback): if not self.result.ready() and not self._waiting: self._waiting = True Greenlet.spawn(self._wait) self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _blocking_wait(self): WaitForSingleObject(self._handle, INFINITE) self.returncode = GetExitCodeProcess(self._handle) return self.returncode def _wait(self): self.threadpool.spawn(self._blocking_wait).rawlink(self.result) def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: if not self._waiting: self._waiting = True self._wait() return self.result.wait(timeout=timeout) else: # # POSIX methods # def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" return self.result.wait(timeout=timeout)
class Popen(object): def __init__(self, args, bufsize=None, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, threadpool=None, **kwargs): """Create new Popen instance.""" if not PY3 and kwargs: raise TypeError("Got unexpected keyword arguments", kwargs) pass_fds = kwargs.pop('pass_fds', ()) start_new_session = kwargs.pop('start_new_session', False) restore_signals = kwargs.pop('restore_signals', True) hub = get_hub() if bufsize is None: # bufsize has different defaults on Py3 and Py2 if PY3: bufsize = -1 else: bufsize = 0 if not isinstance(bufsize, integer_types): raise TypeError("bufsize must be an integer") if mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") any_stdio_set = (stdin is not None or stdout is not None or stderr is not None) if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: if any_stdio_set: close_fds = False else: close_fds = True elif close_fds and any_stdio_set: raise ValueError("close_fds is not supported on Windows " "platforms if you redirect stdin/stdout/stderr") if threadpool is None: threadpool = hub.threadpool self.threadpool = threadpool self._waiting = False else: # POSIX if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: # close_fds has different defaults on Py3/Py2 if PY3: close_fds = True else: close_fds = False if pass_fds and not close_fds: import warnings warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) close_fds = True if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") if creationflags != 0: raise ValueError("creationflags is only supported on Windows " "platforms") assert threadpool is None self._loop = hub.loop if PY3: self.args = args self.stdin = None self.stdout = None self.stderr = None self.pid = None self.returncode = None self.universal_newlines = universal_newlines self.result = AsyncResult() # Input and output objects. The general principle is like # this: # # Parent Child # ------ ----- # p2cwrite ---stdin---> p2cread # c2pread <--stdout--- c2pwrite # errread <--stderr--- errwrite # # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects # are None when not using PIPEs. The child objects are None # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) # We wrap OS handles *before* launching the child, otherwise a # quickly terminating child could make our fds unwrappable # (see #8458). if mswindows: if p2cwrite is not None: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) if c2pread is not None: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) if errread is not None: errread = msvcrt.open_osfhandle(errread.Detach(), 0) if p2cwrite is not None: if PY3 and universal_newlines: # Under Python 3, if we left on the 'b' we'd get different results # depending on whether we used FileObjectPosix or FileObjectThread self.stdin = FileObject(p2cwrite, 'wb', bufsize) self.stdin._translate = True self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True, line_buffering=(bufsize == 1)) else: self.stdin = FileObject(p2cwrite, 'wb', bufsize) if c2pread is not None: if universal_newlines: if PY3: # FileObjectThread doesn't support the 'U' qualifier # with a bufsize of 0 self.stdout = FileObject(c2pread, 'rb', bufsize) # NOTE: Universal Newlines are broken on Windows/Py3, at least # in some cases. This is true in the stdlib subprocess module # as well; the following line would fix the test cases in # test__subprocess.py that depend on python_universal_newlines, # but would be inconsistent with the stdlib: #msvcrt.setmode(self.stdout.fileno(), os.O_TEXT) self.stdout.io = io.TextIOWrapper(self.stdout.io) self.stdout.io.mode = 'r' self.stdout._translate = True else: self.stdout = FileObject(c2pread, 'rU', bufsize) else: self.stdout = FileObject(c2pread, 'rb', bufsize) if errread is not None: if universal_newlines: if PY3: self.stderr = FileObject(errread, 'rb', bufsize) self.stderr.io = io.TextIOWrapper(self.stderr.io) self.stderr._translate = True else: self.stderr = FileObject(errread, 'rU', bufsize) else: self.stderr = FileObject(errread, 'rb', bufsize) self._closed_child_pipe_fds = False try: self._execute_child(args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session) except: # Cleanup if the child failed starting. # (gevent: New in python3, but reported as gevent bug in #347. # Note that under Py2, any error raised below will replace the # original error so we have to use reraise) if not PY3: exc_info = sys.exc_info() for f in filter(None, (self.stdin, self.stdout, self.stderr)): try: f.close() except (OSError, IOError): pass # Ignore EBADF or other errors. if not self._closed_child_pipe_fds: to_close = [] if stdin == PIPE: to_close.append(p2cread) if stdout == PIPE: to_close.append(c2pwrite) if stderr == PIPE: to_close.append(errwrite) if hasattr(self, '_devnull'): to_close.append(self._devnull) for fd in to_close: try: os.close(fd) except (OSError, IOError): pass if not PY3: try: reraise(*exc_info) finally: del exc_info raise def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode) def _on_child(self, watcher): watcher.stop() status = watcher.rstatus if os.WIFSIGNALED(status): self.returncode = -os.WTERMSIG(status) else: self.returncode = os.WEXITSTATUS(status) self.result.set(self.returncode) def _get_devnull(self): if not hasattr(self, '_devnull'): self._devnull = os.open(os.devnull, os.O_RDWR) return self._devnull _stdout_buffer = None _stderr_buffer = None def communicate(self, input=None, timeout=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional input argument should be a string to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr). :keyword timeout: Under Python 2, this is a gevent extension; if given and it expires, we will raise :class:`gevent.timeout.Timeout`. Under Python 3, this raises the standard :exc:`TimeoutExpired` exception. """ greenlets = [] if self.stdin: greenlets.append(spawn(write_and_close, self.stdin, input)) # If the timeout parameter is used, and the caller calls back after # getting a TimeoutExpired exception, we can wind up with multiple # greenlets trying to run and read from and close stdout/stderr. # That's bad because it can lead to 'RuntimeError: reentrant call in io.BufferedReader'. # We can't just kill the previous greenlets when a timeout happens, # though, because we risk losing the output collected by that greenlet # (and Python 3, where timeout is an official parameter, explicitly says # that no output should be lost in the event of a timeout.) Instead, we're # watching for the exception and ignoring it. It's not elegant, # but it works if self.stdout: def _read_out(): try: data = self.stdout.read() except RuntimeError: return if self._stdout_buffer is not None: self._stdout_buffer += data else: self._stdout_buffer = data stdout = spawn(_read_out) greenlets.append(stdout) else: stdout = None if self.stderr: def _read_err(): try: data = self.stderr.read() except RuntimeError: return if self._stderr_buffer is not None: self._stderr_buffer += data else: self._stderr_buffer = data stderr = spawn(_read_err) greenlets.append(stderr) else: stderr = None # If we were given stdin=stdout=stderr=None, we have no way to # communicate with the child, and thus no greenlets to wait # on. This is a nonsense case, but it comes up in the test # case for Python 3.5 (test_subprocess.py # RunFuncTestCase.test_timeout). Instead, we go directly to # self.wait if not greenlets and timeout is not None: result = self.wait(timeout=timeout) # Python 3 would have already raised, but Python 2 would not # so we need to do that manually if result is None: from gevent.timeout import Timeout raise Timeout(timeout) done = joinall(greenlets, timeout=timeout) if timeout is not None and len(done) != len(greenlets): if PY3: raise TimeoutExpired(self.args, timeout) from gevent.timeout import Timeout raise Timeout(timeout) if self.stdout: try: self.stdout.close() except RuntimeError: pass if self.stderr: try: self.stderr.close() except RuntimeError: pass self.wait() stdout_value = self._stdout_buffer self._stdout_buffer = None stderr_value = self._stderr_buffer self._stderr_buffer = None # XXX: Under python 3 in universal newlines mode we should be # returning str, not bytes return (None if stdout is None else stdout_value or b'', None if stderr is None else stderr_value or b'') def poll(self): return self._internal_poll() if PY3: def __enter__(self): return self def __exit__(self, type, value, traceback): if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() try: # Flushing a BufferedWriter may raise an error if self.stdin: self.stdin.close() finally: # Wait for the process to terminate, to avoid zombies. # JAM: gevent: If the process never terminates, this # blocks forever. self.wait() if mswindows: # # Windows methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: return (None, None, None, None, None, None) p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None try: DEVNULL except NameError: _devnull = object() else: _devnull = DEVNULL if stdin is None: p2cread = GetStdHandle(STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = CreatePipe(None, 0) if PY3: p2cread = Handle(p2cread) _winapi.CloseHandle(_) elif stdin == PIPE: p2cread, p2cwrite = CreatePipe(None, 0) if PY3: p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) elif stdin == _devnull: p2cread = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) if stdout is None: c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = CreatePipe(None, 0) if PY3: c2pwrite = Handle(c2pwrite) _winapi.CloseHandle(_) elif stdout == PIPE: c2pread, c2pwrite = CreatePipe(None, 0) if PY3: c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) elif stdout == _devnull: c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) if stderr is None: errwrite = GetStdHandle(STD_ERROR_HANDLE) if errwrite is None: _, errwrite = CreatePipe(None, 0) if PY3: errwrite = Handle(errwrite) _winapi.CloseHandle(_) elif stderr == PIPE: errread, errwrite = CreatePipe(None, 0) if PY3: errread, errwrite = Handle(errread), Handle(errwrite) elif stderr == STDOUT: errwrite = c2pwrite elif stderr == _devnull: errwrite = msvcrt.get_osfhandle(self._get_devnull()) elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" return DuplicateHandle(GetCurrentProcess(), handle, GetCurrentProcess(), 0, 1, DUPLICATE_SAME_ACCESS) def _find_w9xpopen(self): """Find and return absolut path to w9xpopen.exe""" w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), "w9xpopen.exe") if not os.path.exists(w9xpopen): # Eeek - file-not-found - possibly an embedding # situation - see if we can locate it in sys.exec_prefix w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), "w9xpopen.exe") if not os.path.exists(w9xpopen): raise RuntimeError("Cannot locate w9xpopen.exe, which is " "needed for Popen to work with your " "shell or platform.") return w9xpopen def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, unused_restore_signals, unused_start_new_session): """Execute program (MS Windows version)""" assert not pass_fds, "pass_fds not supported on Windows." if not isinstance(args, string_types): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() if None not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite if shell: startupinfo.dwFlags |= STARTF_USESHOWWINDOW startupinfo.wShowWindow = SW_HIDE comspec = os.environ.get("COMSPEC", "cmd.exe") args = '{} /c "{}"'.format(comspec, args) if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com": # Win9x, or using command.com on NT. We need to # use the w9xpopen intermediate program. For more # information, see KB Q150956 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) w9xpopen = self._find_w9xpopen() args = '"%s" %s' % (w9xpopen, args) # Not passing CREATE_NEW_CONSOLE has been known to # cause random failures on win9x. Specifically a # dialog: "Your program accessed mem currently in # use at xxx" and a hopeful warning about the # stability of your system. Cost is Ctrl+C wont # kill children. creationflags |= CREATE_NEW_CONSOLE # Start the process try: hp, ht, pid, tid = CreateProcess(executable, args, # no special security None, None, int(not close_fds), creationflags, env, cwd, startupinfo) except IOError as e: # From 2.6 on, pywintypes.error was defined as IOError # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really # translate errno using _sys_errlist (or similar), but # how can this be done from Python? if PY3: raise # don't remap here raise WindowsError(*e.args) finally: # Child is launched. Close the parent's copy of those pipe # handles that only the child should have open. You need # to make sure that no handles to the write end of the # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. def _close(x): if x is not None and x != -1: if hasattr(x, 'Close'): x.Close() else: _winapi.CloseHandle(x) _close(p2cread) _close(c2pwrite) _close(errwrite) if hasattr(self, '_devnull'): os.close(self._devnull) # Retain the process handle, but close the thread handle self._child_created = True self._handle = Handle(hp) if not hasattr(hp, 'Close') else hp self.pid = pid _winapi.CloseHandle(ht) if not hasattr(ht, 'Close') else ht.Close() def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: self.returncode = GetExitCodeProcess(self._handle) self.result.set(self.returncode) return self.returncode def rawlink(self, callback): if not self.result.ready() and not self._waiting: self._waiting = True Greenlet.spawn(self._wait) self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _blocking_wait(self): WaitForSingleObject(self._handle, INFINITE) self.returncode = GetExitCodeProcess(self._handle) return self.returncode def _wait(self): self.threadpool.spawn(self._blocking_wait).rawlink(self.result) def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: if not self._waiting: self._waiting = True self._wait() result = self.result.wait(timeout=timeout) if PY3 and timeout is not None and not self.result.ready(): raise TimeoutExpired(self.args, timeout) return result def send_signal(self, sig): """Send a signal to the process """ if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: os.kill(self.pid, signal.CTRL_C_EVENT) elif sig == signal.CTRL_BREAK_EVENT: os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process """ TerminateProcess(self._handle, 1) kill = terminate else: # # POSIX methods # def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None try: DEVNULL except NameError: _devnull = object() else: _devnull = DEVNULL if stdin is None: pass elif stdin == PIPE: p2cread, p2cwrite = self.pipe_cloexec() elif stdin == _devnull: p2cread = self._get_devnull() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == PIPE: c2pread, c2pwrite = self.pipe_cloexec() elif stdout == _devnull: c2pwrite = self._get_devnull() elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == PIPE: errread, errwrite = self.pipe_cloexec() elif stderr == STDOUT: errwrite = c2pwrite elif stderr == _devnull: errwrite = self._get_devnull() elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) if cloexec: fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) else: fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _remove_nonblock_flag(self, fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK) fcntl.fcntl(fd, fcntl.F_SETFL, flags) def pipe_cloexec(self): """Create a pipe with FDs set CLOEXEC.""" # Pipes' FDs are set CLOEXEC by default because we don't want them # to be inherited by other subprocesses: the CLOEXEC flag is removed # from the child's FDs by _dup2(), between fork() and exec(). # This is not atomic: we would need the pipe2() syscall for that. r, w = os.pipe() self._set_cloexec_flag(r) self._set_cloexec_flag(w) return r, w def _close_fds(self, keep): # `keep` is a set of fds, so we # use os.closerange from 3 to min(keep) # and then from max(keep + 1) to MAXFD and # loop through filling in the gaps. # Under new python versions, we need to explicitly set # passed fds to be inheritable or they will go away on exec if hasattr(os, 'set_inheritable'): set_inheritable = os.set_inheritable else: set_inheritable = lambda i, v: True if hasattr(os, 'closerange'): keep = sorted(keep) min_keep = min(keep) max_keep = max(keep) os.closerange(3, min_keep) os.closerange(max_keep + 1, MAXFD) for i in xrange(min_keep, max_keep): if i in keep: set_inheritable(i, True) continue try: os.close(i) except: pass else: for i in xrange(3, MAXFD): if i in keep: set_inheritable(i, True) continue try: os.close(i) except: pass def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session): """Execute program (POSIX version)""" if PY3 and isinstance(args, (str, bytes)): args = [args] elif not PY3 and isinstance(args, string_types): args = [args] else: args = list(args) if shell: args = ["/bin/sh", "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] self._loop.install_sigchld() # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = self.pipe_cloexec() # errpipe_write must not be in the standard io 0, 1, or 2 fd range. low_fds_to_close = [] while errpipe_write < 3: low_fds_to_close.append(errpipe_write) errpipe_write = os.dup(errpipe_write) for low_fd in low_fds_to_close: os.close(low_fd) try: try: gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = fork_and_watch(self._on_child, self._loop, True, fork) except: if gc_was_enabled: gc.enable() raise if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # When duping fds, if there arises a situation # where one of the fds is either 0, 1 or 2, it # is possible that it is overwritten (#12607). if c2pwrite == 0: c2pwrite = os.dup(c2pwrite) if errwrite == 0 or errwrite == 1: errwrite = os.dup(errwrite) # Dup fds for child def _dup2(a, b): # dup2() removes the CLOEXEC flag but # we must do it ourselves if dup2() # would be a no-op (issue #10806). if a == b: self._set_cloexec_flag(a, False) elif a is not None: os.dup2(a, b) self._remove_nonblock_flag(b) _dup2(p2cread, 0) _dup2(c2pwrite, 1) _dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the # same fd more than once, or standard fds. closed = set([None]) for fd in [p2cread, c2pwrite, errwrite]: if fd not in closed and fd > 2: os.close(fd) closed.add(fd) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() # Close all other fds, if asked for. This must be done # after preexec_fn runs. if close_fds: fds_to_keep = set(pass_fds) fds_to_keep.add(errpipe_write) self._close_fds(fds_to_keep) elif hasattr(os, 'get_inheritable'): # close_fds was false, and we're on # Python 3.4 or newer, so "all file # descriptors except standard streams # are closed, and inheritable handles # are only inherited if the close_fds # parameter is False." for i in xrange(3, MAXFD): try: if i == errpipe_write or os.get_inheritable(i): continue os.close(i) except: pass if restore_signals: # restore the documented signals back to sig_dfl; # not all will be defined on every platform for sig in 'SIGPIPE', 'SIGXFZ', 'SIGXFSZ': sig = getattr(signal, sig, None) if sig is not None: signal.signal(sig, signal.SIG_DFL) if start_new_session: os.setsid() if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception(exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) finally: # Make sure that the process exits no matter what. # The return code does not matter much as it won't be # reported to the application os._exit(1) # Parent self._child_created = True if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) # self._devnull is not always defined. devnull_fd = getattr(self, '_devnull', None) if p2cread is not None and p2cwrite is not None and p2cread != devnull_fd: os.close(p2cread) if c2pwrite is not None and c2pread is not None and c2pwrite != devnull_fd: os.close(c2pwrite) if errwrite is not None and errread is not None and errwrite != devnull_fd: os.close(errwrite) if devnull_fd is not None: os.close(devnull_fd) # Prevent a double close of these fds from __init__ on error. self._closed_child_pipe_fds = True # Wait for exec to fail or succeed; possibly raising exception errpipe_read = FileObject(errpipe_read, 'rb') data = errpipe_read.read() finally: if hasattr(errpipe_read, 'close'): errpipe_read.close() else: os.close(errpipe_read) if data != b"": self.wait() child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception def _handle_exitstatus(self, sts): if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) elif os.WIFEXITED(sts): self.returncode = os.WEXITSTATUS(sts) else: # Should never happen raise RuntimeError("Unknown child exit status!") def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute. :keyword timeout: The floating point number of seconds to wait. Under Python 2, this is a gevent extension, and we simply return if it expires. Under Python 3, if this time elapses without finishing the process, :exc:`TimeoutExpired` is raised.""" result = self.result.wait(timeout=timeout) if PY3 and timeout is not None and not self.result.ready(): raise TimeoutExpired(self.args, timeout) return result def send_signal(self, sig): """Send a signal to the process """ os.kill(self.pid, sig) def terminate(self): """Terminate the process with SIGTERM """ self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ self.send_signal(signal.SIGKILL)
class AlarmTask(Runnable): """ Task to notify when a block is mined. """ def __init__(self, chain): super().__init__() self.callbacks = list() self.chain = chain self.chain_id = None self.known_block_number = None # TODO: Start with a larger sleep_time and decrease it as the # probability of a new block increases. self.sleep_time = 0.5 def start(self): log.debug('Alarm task started', node=pex(self.chain.node_address)) self._stop_event = AsyncResult() super().start() def _run(self): # pylint: disable=method-hidden try: self.loop_until_stop() finally: self.callbacks = list() def register_callback(self, callback): """ Register a new callback. Note: The callback will be executed in the AlarmTask context and for this reason it should not block, otherwise we can miss block changes. """ if not callable(callback): raise ValueError('callback is not a callable') self.callbacks.append(callback) def remove_callback(self, callback): """Remove callback from the list of callbacks if it exists""" if callback in self.callbacks: self.callbacks.remove(callback) def loop_until_stop(self): # The AlarmTask must have completed its first_run() before starting # the background greenlet. # # This is required because the first run will synchronize the node with # the blockchain since the last run. assert self.chain_id, 'chain_id not set' assert self.known_block_number is not None, 'known_block_number not set' sleep_time = self.sleep_time while self._stop_event.wait(sleep_time) is not True: try: latest_block = self.chain.get_block(block_identifier='latest') except JSONDecodeError as e: raise EthNodeCommunicationError(str(e)) self._maybe_run_callbacks(latest_block) def first_run(self, known_block_number): """ Blocking call to update the local state, if necessary. """ assert self.callbacks, 'callbacks not set' latest_block = self.chain.get_block(block_identifier='latest') log.debug( 'Alarm task first run', known_block_number=known_block_number, latest_block_number=latest_block['number'], latest_gas_limit=latest_block['gasLimit'], latest_block_hash=to_hex(latest_block['hash']), ) self.known_block_number = known_block_number self.chain_id = self.chain.network_id self._maybe_run_callbacks(latest_block) def _maybe_run_callbacks(self, latest_block): """ Run the callbacks if there is at least one new block. The callbacks are executed only if there is a new block, otherwise the filters may try to poll for an inexisting block number and the Ethereum client can return an JSON-RPC error. """ assert self.known_block_number is not None, 'known_block_number not set' latest_block_number = latest_block['number'] missed_blocks = latest_block_number - self.known_block_number if missed_blocks < 0: log.critical( 'Block number decreased', chain_id=self.chain_id, known_block_number=self.known_block_number, old_block_number=latest_block['number'], old_gas_limit=latest_block['gasLimit'], old_block_hash=to_hex(latest_block['hash']), ) elif missed_blocks > 0: log_details = dict( known_block_number=self.known_block_number, latest_block_number=latest_block_number, latest_block_hash=to_hex(latest_block['hash']), latest_block_gas_limit=latest_block['gasLimit'], ) if missed_blocks > 1: log_details['num_missed_blocks'] = missed_blocks - 1 log.debug( 'Received new block', **log_details, ) remove = list() for callback in self.callbacks: result = callback(latest_block) if result is REMOVE_CALLBACK: remove.append(callback) for callback in remove: self.callbacks.remove(callback) self.known_block_number = latest_block_number def stop(self): self._stop_event.set(True) log.debug('Alarm task stopped', node=pex(self.chain.node_address)) result = self.join() # Callbacks should be cleaned after join self.callbacks = [] return result
class Popen(object): # copy of gevent.subprocess.Popen; won't be needed once gevent's subprocess supports pass_fds def __init__(self, args, pass_fds=()): hub = get_hub() self._loop = hub.loop self.pid = None self.returncode = None self.result = AsyncResult() self._execute_child(args, pass_fds) def __repr__(self): return "<%s at 0x%x pid=%r returncode=%r>" % (self.__class__.__name__, id(self), self.pid, self.returncode) def _on_child(self, watcher): watcher.stop() status = watcher.rstatus if os.WIFSIGNALED(status): self.returncode = -os.WTERMSIG(status) else: self.returncode = os.WEXITSTATUS(status) self.result.set(self.returncode) def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) def _execute_child(self, args, pass_fds): executable = args[0] self._loop.install_sigchld() gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = fork() except: if gc_was_enabled: gc.enable() raise if self.pid == 0: _close_fds(pass_fds) try: os.execvp(executable, args) finally: os._exit(1) # Parent self._watcher = self._loop.child(self.pid) self._watcher.start(self._on_child, self._watcher) if gc_was_enabled: gc.enable() def poll(self): if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, "sig_pending", True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): return self.result.wait(timeout=timeout) def send_signal(self, sig): os.kill(self.pid, sig)
class AlarmTask(Task): """ Task to notify when a block is mined. """ def __init__(self, chain): super(AlarmTask, self).__init__() self.callbacks = list() self.stop_event = AsyncResult() self.chain = chain self.last_block_number = None # TODO: Start with a larger wait_time and decrease it as the # probability of a new block increases. self.wait_time = 0.5 self.last_loop = time.time() def register_callback(self, callback): """ Register a new callback. Note: The callback will be executed in the AlarmTask context and for this reason it should not block, otherwise we can miss block changes. """ if not callable(callback): raise ValueError('callback is not a callable') self.callbacks.append(callback) def remove_callback(self, callback): """Remove callback from the list of callbacks if it exists""" if callback in self.callbacks: self.callbacks.remove(callback) def _run(self): # pylint: disable=method-hidden log.debug('starting block number', block_number=self.last_block_number) sleep_time = 0 while self.stop_event.wait(sleep_time) is not True: try: self.poll_for_new_block() except RaidenShuttingDown: break # we want this task to iterate in the tick of `wait_time`, so take # into account how long we spent executing one tick. self.last_loop = time.time() work_time = self.last_loop - self.last_loop if work_time > self.wait_time: log.warning( 'alarm loop is taking longer than the wait time', work_time=work_time, wait_time=self.wait_time, ) sleep_time = 0.001 else: sleep_time = self.wait_time - work_time # stopping self.callbacks = list() def poll_for_new_block(self): current_block = self.chain.block_number() if current_block > self.last_block_number + 1: difference = current_block - self.last_block_number - 1 log.error( 'alarm missed %s blocks', difference, ) if current_block != self.last_block_number: log.debug( 'new block', number=current_block, timestamp=self.last_loop, ) self.last_block_number = current_block remove = list() for callback in self.callbacks: try: result = callback(current_block) except RaidenShuttingDown: break except: # pylint: disable=bare-except # noqa log.exception('unexpected exception on alarm') else: if result is REMOVE_CALLBACK: remove.append(callback) for callback in remove: self.callbacks.remove(callback) def start(self): self.last_block_number = self.chain.block_number() super(AlarmTask, self).start() def stop_and_wait(self): self.stop_event.set(True) gevent.wait(self) def stop_async(self): self.stop_event.set(True)
class Popen(object): def __init__(self, args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, threadpool=None): """Create new Popen instance.""" if not isinstance(bufsize, integer_types): raise TypeError("bufsize must be an integer") hub = get_hub() if mswindows: if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") if close_fds and (stdin is not None or stdout is not None or stderr is not None): raise ValueError("close_fds is not supported on Windows " "platforms if you redirect stdin/stdout/stderr") if threadpool is None: threadpool = hub.threadpool self.threadpool = threadpool self._waiting = False else: # POSIX if startupinfo is not None: raise ValueError("startupinfo is only supported on Windows " "platforms") if creationflags != 0: raise ValueError("creationflags is only supported on Windows " "platforms") assert threadpool is None self._loop = hub.loop self.stdin = None self.stdout = None self.stderr = None self.pid = None self.returncode = None self.universal_newlines = universal_newlines self.result = AsyncResult() # Input and output objects. The general principle is like # this: # # Parent Child # ------ ----- # p2cwrite ---stdin---> p2cread # c2pread <--stdout--- c2pwrite # errread <--stderr--- errwrite # # On POSIX, the child objects are file descriptors. On # Windows, these are Windows file handles. The parent objects # are file descriptors on both platforms. The parent objects # are None when not using PIPEs. The child objects are None # when not redirecting. (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) = self._get_handles(stdin, stdout, stderr) self._execute_child(args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) if mswindows: if p2cwrite is not None: p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) if c2pread is not None: c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) if errread is not None: errread = msvcrt.open_osfhandle(errread.Detach(), 0) if p2cwrite is not None: self.stdin = FileObject(p2cwrite, 'wb') if c2pread is not None: if universal_newlines: self.stdout = FileObject(c2pread, 'rU') else: self.stdout = FileObject(c2pread, 'rb') if errread is not None: if universal_newlines: self.stderr = FileObject(errread, 'rU') else: self.stderr = FileObject(errread, 'rb') def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode) def _on_child(self, watcher): watcher.stop() status = watcher.rstatus if os.WIFSIGNALED(status): self.returncode = -os.WTERMSIG(status) else: self.returncode = os.WEXITSTATUS(status) self.result.set(self.returncode) def communicate(self, input=None): """Interact with process: Send data to stdin. Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. The optional input argument should be a string to be sent to the child process, or None, if no data should be sent to the child. communicate() returns a tuple (stdout, stderr).""" greenlets = [] if self.stdin: greenlets.append(spawn(write_and_close, self.stdin, input)) if self.stdout: stdout = spawn(self.stdout.read) greenlets.append(stdout) else: stdout = None if self.stderr: stderr = spawn(self.stderr.read) greenlets.append(stderr) else: stderr = None joinall(greenlets) if self.stdout: self.stdout.close() if self.stderr: self.stderr.close() self.wait() return (None if stdout is None else stdout.value or '', None if stderr is None else stderr.value or '') def poll(self): return self._internal_poll() if mswindows: # # Windows methods # def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ if stdin is None and stdout is None and stderr is None: return (None, None, None, None, None, None) p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None if stdin is None: p2cread = GetStdHandle(STD_INPUT_HANDLE) if p2cread is None: p2cread, _ = CreatePipe(None, 0) elif stdin == PIPE: p2cread, p2cwrite = CreatePipe(None, 0) elif isinstance(stdin, int): p2cread = msvcrt.get_osfhandle(stdin) else: # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) if stdout is None: c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) if c2pwrite is None: _, c2pwrite = CreatePipe(None, 0) elif stdout == PIPE: c2pread, c2pwrite = CreatePipe(None, 0) elif isinstance(stdout, int): c2pwrite = msvcrt.get_osfhandle(stdout) else: # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) if stderr is None: errwrite = GetStdHandle(STD_ERROR_HANDLE) if errwrite is None: _, errwrite = CreatePipe(None, 0) elif stderr == PIPE: errread, errwrite = CreatePipe(None, 0) elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): errwrite = msvcrt.get_osfhandle(stderr) else: # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _make_inheritable(self, handle): """Return a duplicate of handle, which is inheritable""" return DuplicateHandle(GetCurrentProcess(), handle, GetCurrentProcess(), 0, 1, DUPLICATE_SAME_ACCESS) def _find_w9xpopen(self): """Find and return absolut path to w9xpopen.exe""" w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), "w9xpopen.exe") if not os.path.exists(w9xpopen): # Eeek - file-not-found - possibly an embedding # situation - see if we can locate it in sys.exec_prefix w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), "w9xpopen.exe") if not os.path.exists(w9xpopen): raise RuntimeError("Cannot locate w9xpopen.exe, which is " "needed for Popen to work with your " "shell or platform.") return w9xpopen def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (MS Windows version)""" if not isinstance(args, string_types): args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() if None not in (p2cread, c2pwrite, errwrite): startupinfo.dwFlags |= STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite if shell: startupinfo.dwFlags |= STARTF_USESHOWWINDOW startupinfo.wShowWindow = SW_HIDE comspec = os.environ.get("COMSPEC", "cmd.exe") args = '{} /c "{}"'.format(comspec, args) if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com": # Win9x, or using command.com on NT. We need to # use the w9xpopen intermediate program. For more # information, see KB Q150956 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) w9xpopen = self._find_w9xpopen() args = '"%s" %s' % (w9xpopen, args) # Not passing CREATE_NEW_CONSOLE has been known to # cause random failures on win9x. Specifically a # dialog: "Your program accessed mem currently in # use at xxx" and a hopeful warning about the # stability of your system. Cost is Ctrl+C wont # kill children. creationflags |= CREATE_NEW_CONSOLE # Start the process try: hp, ht, pid, tid = CreateProcess(executable, args, # no special security None, None, int(not close_fds), creationflags, env, cwd, startupinfo) except pywintypes.error as e: # Translate pywintypes.error to WindowsError, which is # a subclass of OSError. FIXME: We should really # translate errno using _sys_errlist (or similar), but # how can this be done from Python? raise WindowsError(*e.args) finally: # Child is launched. Close the parent's copy of those pipe # handles that only the child should have open. You need # to make sure that no handles to the write end of the # output pipe are maintained in this process or else the # pipe will not close when the child process exits and the # ReadFile will hang. if p2cread is not None: p2cread.Close() if c2pwrite is not None: c2pwrite.Close() if errwrite is not None: errwrite.Close() # Retain the process handle, but close the thread handle self._handle = hp self.pid = pid ht.Close() def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: self.returncode = GetExitCodeProcess(self._handle) self.result.set(self.returncode) return self.returncode def rawlink(self, callback): if not self.result.ready() and not self._waiting: self._waiting = True Greenlet.spawn(self._wait) self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _blocking_wait(self): WaitForSingleObject(self._handle, INFINITE) self.returncode = GetExitCodeProcess(self._handle) return self.returncode def _wait(self): self.threadpool.spawn(self._blocking_wait).rawlink(self.result) def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" if self.returncode is None: if not self._waiting: self._waiting = True self._wait() return self.result.wait(timeout=timeout) def send_signal(self, sig): """Send a signal to the process """ if sig == signal.SIGTERM: self.terminate() elif sig == signal.CTRL_C_EVENT: os.kill(self.pid, signal.CTRL_C_EVENT) elif sig == signal.CTRL_BREAK_EVENT: os.kill(self.pid, signal.CTRL_BREAK_EVENT) else: raise ValueError("Unsupported signal: {}".format(sig)) def terminate(self): """Terminates the process """ TerminateProcess(self._handle, 1) kill = terminate else: # # POSIX methods # def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) # XXX unlink def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None if stdin is None: pass elif stdin == PIPE: p2cread, p2cwrite = self.pipe_cloexec() elif isinstance(stdin, int): p2cread = stdin else: # Assuming file-like object p2cread = stdin.fileno() if stdout is None: pass elif stdout == PIPE: c2pread, c2pwrite = self.pipe_cloexec() elif isinstance(stdout, int): c2pwrite = stdout else: # Assuming file-like object c2pwrite = stdout.fileno() if stderr is None: pass elif stderr == PIPE: errread, errwrite = self.pipe_cloexec() elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): errwrite = stderr else: # Assuming file-like object errwrite = stderr.fileno() return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) def _set_cloexec_flag(self, fd, cloexec=True): try: cloexec_flag = fcntl.FD_CLOEXEC except AttributeError: cloexec_flag = 1 old = fcntl.fcntl(fd, fcntl.F_GETFD) if cloexec: fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) else: fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) def _remove_nonblock_flag(self, fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK) fcntl.fcntl(fd, fcntl.F_SETFL, flags) def pipe_cloexec(self): """Create a pipe with FDs set CLOEXEC.""" # Pipes' FDs are set CLOEXEC by default because we don't want them # to be inherited by other subprocesses: the CLOEXEC flag is removed # from the child's FDs by _dup2(), between fork() and exec(). # This is not atomic: we would need the pipe2() syscall for that. r, w = os.pipe() self._set_cloexec_flag(r) self._set_cloexec_flag(w) return r, w def _close_fds(self, but): if hasattr(os, 'closerange'): os.closerange(3, but) os.closerange(but + 1, MAXFD) else: for i in xrange(3, MAXFD): if i == but: continue try: os.close(i) except: pass def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): """Execute program (POSIX version)""" if isinstance(args, string_types): args = [args] else: args = list(args) if shell: args = ["/bin/sh", "-c"] + args if executable: args[0] = executable if executable is None: executable = args[0] self._loop.install_sigchld() # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. errpipe_read, errpipe_write = self.pipe_cloexec() try: try: gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = fork() except: if gc_was_enabled: gc.enable() raise if self.pid == 0: # Child try: # Close parent's pipe ends if p2cwrite is not None: os.close(p2cwrite) if c2pread is not None: os.close(c2pread) if errread is not None: os.close(errread) os.close(errpipe_read) # When duping fds, if there arises a situation # where one of the fds is either 0, 1 or 2, it # is possible that it is overwritten (#12607). if c2pwrite == 0: c2pwrite = os.dup(c2pwrite) if errwrite == 0 or errwrite == 1: errwrite = os.dup(errwrite) # Dup fds for child def _dup2(a, b): # dup2() removes the CLOEXEC flag but # we must do it ourselves if dup2() # would be a no-op (issue #10806). if a == b: self._set_cloexec_flag(a, False) elif a is not None: os.dup2(a, b) self._remove_nonblock_flag(b) _dup2(p2cread, 0) _dup2(c2pwrite, 1) _dup2(errwrite, 2) # Close pipe fds. Make sure we don't close the # same fd more than once, or standard fds. closed = set([None]) for fd in [p2cread, c2pwrite, errwrite]: if fd not in closed and fd > 2: os.close(fd) closed.add(fd) # Close all other fds, if asked for if close_fds: self._close_fds(but=errpipe_write) if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() if env is None: os.execvp(executable, args) else: os.execvpe(executable, args, env) except: exc_type, exc_value, tb = sys.exc_info() # Save the traceback and attach it to the exception object exc_lines = traceback.format_exception(exc_type, exc_value, tb) exc_value.child_traceback = ''.join(exc_lines) os.write(errpipe_write, pickle.dumps(exc_value)) finally: # Make sure that the process exits no matter what. # The return code does not matter much as it won't be # reported to the application os._exit(1) # Parent self._watcher = self._loop.child(self.pid) self._watcher.start(self._on_child, self._watcher) if gc_was_enabled: gc.enable() finally: # be sure the FD is closed no matter what os.close(errpipe_write) if p2cread is not None and p2cwrite is not None: os.close(p2cread) if c2pwrite is not None and c2pread is not None: os.close(c2pwrite) if errwrite is not None and errread is not None: os.close(errwrite) # Wait for exec to fail or succeed; possibly raising exception errpipe_read = FileObject(errpipe_read, 'rb') data = errpipe_read.read() finally: if hasattr(errpipe_read, 'close'): errpipe_read.close() else: os.close(errpipe_read) if data != b"": self.wait() child_exception = pickle.loads(data) for fd in (p2cwrite, c2pread, errread): if fd is not None: os.close(fd) raise child_exception def _handle_exitstatus(self, sts): if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) elif os.WIFEXITED(sts): self.returncode = os.WEXITSTATUS(sts) else: # Should never happen raise RuntimeError("Unknown child exit status!") def _internal_poll(self): """Check if child process has terminated. Returns returncode attribute. """ if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): """Wait for child process to terminate. Returns returncode attribute.""" return self.result.wait(timeout=timeout) def send_signal(self, sig): """Send a signal to the process """ os.kill(self.pid, sig) def terminate(self): """Terminate the process with SIGTERM """ self.send_signal(signal.SIGTERM) def kill(self): """Kill the process with SIGKILL """ self.send_signal(signal.SIGKILL)
class ThreadManager(object): """ @brief Manage spawning threads of multiple kinds and ensure they're alive. TODO: Add heartbeats with zeromq for monitoring and restarting. """ def __init__(self, heartbeat_secs=10.0, failure_notify_callback=None): """ Creates a ThreadManager. @param heartbeat_secs Seconds between heartbeats. @param failure_notify_callback Callback to execute when a child fails unexpectedly. Should be a callable taking two params: this process supervisor, and the thread that failed. """ super(ThreadManager, self).__init__() # NOTE: Assumes that pids never overlap between the various process types self.children = [] self.heartbeat_secs = heartbeat_secs self._shutting_down = False self._failure_notify_callback = failure_notify_callback self._shutdown_event = AsyncResult() def _create_thread(self, target=None, **kwargs): """ Creates a "thread" of the proper type. """ return PyonThread(target=target, **kwargs) def spawn(self, target=None, **kwargs): """ @brief Spawn a pyon thread """ log.debug("ThreadManager.spawn, target=%s, kwargs=%s", target, kwargs) proc = self._create_thread(target=target, **kwargs) proc.supervisor = self proc.start() self.children.append(proc) # install failure monitor proc.proc.link_exception(self._child_failed) return proc def _child_failed(self, gproc): # extract any PyonThreadTracebacks - one should be last extra = "" if len(gproc.exception.args) and isinstance(gproc.exception.args[-1], PyonThreadTraceback): extra = "\n" + str(gproc.exception.args[-1]) log.error("Child failed with an exception: (%s) %s%s", gproc, gproc.exception, extra) if self._failure_notify_callback: self._failure_notify_callback(gproc) def ensure_ready(self, proc, errmsg=None, timeout=10): """ Waits until either the thread dies or reports it is ready, whichever comes first. If the thread dies or times out while waiting for it to be ready, a ContainerError is raised. You must be sure the thread implements get_ready_event properly, otherwise this method returns immediately as the base class behavior simply passes. @param proc The thread to wait on. @param errmsg A custom error message to put in the ContainerError's message. May be blank. @param timeout Amount of time (in seconds) to wait for the ready, default 10 seconds. @throws ContainerError If the thread dies or if we get a timeout before the process signals ready. """ if not errmsg: errmsg = "ensure_ready failed" ev = Event() def cb(*args, **kwargs): ev.set() # link either a greenlet failure due to exception OR a success via ready event proc.proc.link_exception(cb) proc.get_ready_event().rawlink(cb) retval = ev.wait(timeout=timeout) # unlink the events: ready event is probably harmless but the exception one, we want to install our own later proc.get_ready_event().unlink(cb) # if the thread is stopped while we are waiting, proc.proc is set to None if proc.proc is not None: proc.proc.unlink(cb) # raise an exception if: # - we timed out # - we caught an exception if not retval: raise ContainerError("%s (timed out)" % errmsg) elif proc.proc is not None and proc.proc.dead and not proc.proc.successful(): raise ContainerError("%s (failed): %s" % (errmsg, proc.proc.exception)) def child_stopped(self, proc): if proc in self.children: # no longer need to listen for exceptions if proc.proc is not None: proc.proc.unlink(self._child_failed) def join_children(self, timeout=None): """ Give child threads "timeout" seconds to shutdown, then forcibly terminate. """ time_start = time.time() child_count = len(self.children) for proc in self.children: # if a child thread has already exited, we don't need to wait on anything - # it's already good to go and can be considered joined. Otherwise we will likely # double call notify_stop which is a bad thing. if proc.proc.dead: continue time_elapsed = time.time() - time_start if timeout is not None: time_remaining = timeout - time_elapsed if time_remaining > 0: # The nice way; let it do cleanup try: proc.notify_stop() proc.join(time_remaining) except Exception: # not playing nice? just kill it. proc.stop() else: # Out of time. Cya, sucker proc.stop() else: proc.join() time_elapsed = time.time() - time_start log.debug('Took %.2fs to shutdown %d child threads', time_elapsed, child_count) return time_elapsed def wait_children(self, timeout=None): """ Performs a join to allow children to complete, then a get() to fetch their results. This will raise an exception if any of the children raises an exception. """ self.join_children(timeout=timeout) return [x.get() for x in self.children] def target(self): try: while not self._shutting_down: self.send_heartbeats() self._shutdown_event.wait(timeout=self.heartbeat_secs) except: log.error('thread died', exc_info=True) def send_heartbeats(self): """ TODO: implement heartbeat and monitors """ #log.debug('lub-dub') pass def shutdown(self, timeout=30.0): """ @brief Give child thread "timeout" seconds to shutdown, then forcibly terminate. """ self._shutting_down = True self._shutdown_event.set(True) unset = shutdown_or_die(timeout) # Failsafe in case the following doesn't work elapsed = self.join_children(timeout) #self.stop() unset() return elapsed
class Popen(object): # copy of gevent.subprocess.Popen; won't be needed once gevent's subprocess supports pass_fds def __init__(self, args, pass_fds=()): hub = get_hub() self._loop = hub.loop self.pid = None self.returncode = None self.result = AsyncResult() self._execute_child(args, pass_fds) def __repr__(self): return '<%s at 0x%x pid=%r returncode=%r>' % ( self.__class__.__name__, id(self), self.pid, self.returncode) def _on_child(self, watcher): watcher.stop() status = watcher.rstatus if os.WIFSIGNALED(status): self.returncode = -os.WTERMSIG(status) else: self.returncode = os.WEXITSTATUS(status) self.result.set(self.returncode) def rawlink(self, callback): self.result.rawlink(linkproxy(callback, self)) def _execute_child(self, args, pass_fds): executable = args[0] self._loop.install_sigchld() gc_was_enabled = gc.isenabled() # Disable gc to avoid bug where gc -> file_dealloc -> # write to stderr -> hang. http://bugs.python.org/issue1336 gc.disable() try: self.pid = fork() except: if gc_was_enabled: gc.enable() raise if self.pid == 0: _close_fds(pass_fds) try: os.execvp(executable, args) finally: os._exit(1) # Parent self._watcher = self._loop.child(self.pid) self._watcher.start(self._on_child, self._watcher) if gc_was_enabled: gc.enable() def poll(self): if self.returncode is None: if get_hub() is not getcurrent(): sig_pending = getattr(self._loop, 'sig_pending', True) if sig_pending: sleep(0.00001) return self.returncode def wait(self, timeout=None): return self.result.wait(timeout=timeout) def send_signal(self, sig): os.kill(self.pid, sig)
class StartMediatedTransferTask(Task): def __init__(self, transfermanager, amount, target, done_result): super(StartMediatedTransferTask, self).__init__() self.amount = amount self.address = transfermanager.assetmanager.raiden.address self.target = target self.transfermanager = transfermanager self.done_result = done_result def __repr__(self): return '<{} {}>'.format( self.__class__.__name__, pex(self.address), ) def _run(self): # pylint: disable=method-hidden,too-many-locals amount = self.amount target = self.target raiden = self.transfermanager.assetmanager.raiden fee = 0 # there are no guarantees that the next_hop will follow the same route routes = self.transfermanager.assetmanager.get_best_routes( amount, target, lock_timeout=None, ) if log.isEnabledFor(logging.DEBUG): log.debug( 'START MEDIATED TRANSFER initiator:%s target:%s', pex(self.address), pex(self.target), ) for path, forward_channel in routes: # try a new secret secret = sha3(hex(random.getrandbits(256))) hashlock = sha3(secret) next_hop = path[1] if log.isEnabledFor(logging.DEBUG): log.debug( 'START MEDIATED TRANSFER NEW PATH path:%s hashlock:%s', lpex(path), pex(hashlock), ) self.transfermanager.register_task_for_hashlock(self, hashlock) lock_expiration = (raiden.chain.block_number() + forward_channel.settle_timeout - raiden.config['reveal_timeout']) mediated_transfer = forward_channel.create_mediatedtransfer( raiden.address, target, fee, amount, lock_expiration, hashlock, ) raiden.sign(mediated_transfer) forward_channel.register_transfer(mediated_transfer) response = self.send_and_wait_valid(raiden, path, mediated_transfer) # `next_hop` timedout if response is None: self.transfermanager.on_hashlock_result(hashlock, False) # someone down the line timedout / couldn't proceed elif isinstance(response, (RefundTransfer, TransferTimeout)): self.transfermanager.on_hashlock_result(hashlock, False) # `target` received the MediatedTransfer elif response.sender == target and isinstance( response, SecretRequest): secret_message = Secret(secret) raiden.sign(secret_message) raiden.send_async(target, secret_message) # register the secret now and just incur with the additional # overhead of retrying until the `next_hop` receives the secret # forward_channel.register_secret(secret) # wait until `next_hop` received the secret to syncronize our # state (otherwise we can send a new transfer with an invalid # locksroot while the secret is in transit that will incur into # additional retry/timeout latency) next_hop = path[1] while True: response = self.response_message.wait() # critical write section self.response_message = AsyncResult() # /critical write section if isinstance(response, Secret) and response.sender == next_hop: # critical read/write section # The channel and it's queue must be locked, a transfer # must not be created while we update the balance_proof. forward_channel.claim_lock(secret) raiden.send_async(next_hop, secret_message) # /critical write section self.transfermanager.on_hashlock_result(hashlock, True) self.done_result.set(True) return log.error( 'Invalid message ignoring. %s', repr(response), ) else: log.error( 'Unexpected response %s', repr(response), ) self.transfermanager.on_hashlock_result(hashlock, False) if log.isEnabledFor(logging.DEBUG): log.debug( 'START MEDIATED TRANSFER FAILED initiator:%s target:%s', pex(self.address), pex(self.target), ) self.done_result.set(False) # all paths failed def send_and_wait_valid(self, raiden, path, mediated_transfer): # pylint: disable=no-self-use """ Send the `mediated_transfer` and wait for either a message from `target` or the `next_hop`. Validate the message received and discards the invalid ones. The most important case being next_hop sending a SecretRequest. """ message_timeout = raiden.config['msg_timeout'] next_hop = path[1] target = path[-1] current_time = time.time() limit_time = current_time + message_timeout # this event is used by the transfermanager to notify the task that a # response was received self.response_message = AsyncResult() raiden.send_async(next_hop, mediated_transfer) while current_time <= limit_time: # wait for a response message (not the Ack for the transfer) response = self.response_message.wait(limit_time - current_time) # reset so that a value can be received either because the current # result was invalid or because we will wait for the next message. # # critical write section self.response_message = AsyncResult() # /critical write section if response is None: if log.isEnabledFor(logging.DEBUG): log.debug( 'MEDIATED TRANSFER TIMED OUT hashlock:%s', pex(mediated_transfer.lock.hashlock), ) return None if response.sender == next_hop: if isinstance(response, (RefundTransfer, TransferTimeout)): return response else: if log.isEnabledFor(logging.INFO): log.info( 'Partner %s sent an invalid message', pex(next_hop), ) return None if response.sender == target: if isinstance(response, SecretRequest): return response else: if log.isEnabledFor(logging.INFO): log.info( 'target %s sent an invalid message', pex(target), ) return None current_time = time.time() if log.isEnabledFor(logging.ERROR): log.error( 'Invalid message ignoring. %s', repr(response), ) return None
class AlarmTask(Runnable): """ Task to notify when a block is mined. """ def __init__(self, chain): super().__init__() self.callbacks = list() self.chain = chain self.chain_id = None self.known_block_number = None self._stop_event = AsyncResult() # TODO: Start with a larger sleep_time and decrease it as the # probability of a new block increases. self.sleep_time = 0.5 def start(self): log.debug('Alarm task started', node=pex(self.chain.node_address)) self._stop_event.set(False) super().start() def _run(self): # pylint: disable=method-hidden try: self.loop_until_stop() finally: self.callbacks = list() def register_callback(self, callback): """ Register a new callback. Note: The callback will be executed in the AlarmTask context and for this reason it should not block, otherwise we can miss block changes. """ if not callable(callback): raise ValueError('callback is not a callable') self.callbacks.append(callback) def remove_callback(self, callback): """Remove callback from the list of callbacks if it exists""" if callback in self.callbacks: self.callbacks.remove(callback) def loop_until_stop(self): # The AlarmTask must have completed its first_run() before starting # the background greenlet. # # This is required because the first run will synchronize the node with # the blockchain since the last run. assert self.chain_id, 'chain_id not set' assert self.known_block_number is not None, 'known_block_number not set' chain_id = self.chain_id sleep_time = self.sleep_time while self._stop_event.wait(sleep_time) is not True: try: latest_block = self.chain.get_block(block_identifier='latest') except JSONDecodeError as e: raise EthNodeCommunicationError(str(e)) self._maybe_run_callbacks(latest_block) if chain_id != self.chain.network_id: raise RuntimeError( 'Changing the underlying blockchain while the Raiden node is running ' 'is not supported.', ) def first_run(self, known_block_number): """ Blocking call to update the local state, if necessary. """ assert self.callbacks, 'callbacks not set' chain_id = self.chain.network_id latest_block = self.chain.get_block(block_identifier='latest') log.debug( 'Alarm task first run', known_block_number=known_block_number, latest_block_number=latest_block['number'], latest_gas_limit=latest_block['gasLimit'], latest_block_hash=to_hex(latest_block['hash']), ) self.known_block_number = known_block_number self.chain_id = chain_id self._maybe_run_callbacks(latest_block) def _maybe_run_callbacks(self, latest_block): """ Run the callbacks if there is at least one new block. The callbacks are executed only if there is a new block, otherwise the filters may try to poll for an inexisting block number and the Ethereum client can return an JSON-RPC error. """ assert self.known_block_number is not None, 'known_block_number not set' latest_block_number = latest_block['number'] missed_blocks = latest_block_number - self.known_block_number if missed_blocks < 0: log.critical( 'Block number decreased', chain_id=self.chain_id, known_block_number=self.known_block_number, old_block_number=latest_block['number'], old_gas_limit=latest_block['gasLimit'], old_block_hash=to_hex(latest_block['hash']), ) elif missed_blocks > 0: log_details = dict( known_block_number=self.known_block_number, latest_block_number=latest_block_number, latest_block_hash=to_hex(latest_block['hash']), latest_block_gas_limit=latest_block['gasLimit'], ) if missed_blocks > 1: log_details['num_missed_blocks'] = missed_blocks - 1 log.debug( 'Received new block', **log_details, ) remove = list() for callback in self.callbacks: result = callback(latest_block) if result is REMOVE_CALLBACK: remove.append(callback) for callback in remove: self.callbacks.remove(callback) self.known_block_number = latest_block_number def stop(self): self._stop_event.set(True) log.debug('Alarm task stopped', node=pex(self.chain.node_address)) result = self.join() # Callbacks should be cleaned after join self.callbacks = [] return result
class EndMediatedTransferTask(Task): """ Task that request a secret for a registered transfer. """ def __init__(self, transfermanager, originating_transfer): super(EndMediatedTransferTask, self).__init__() self.address = transfermanager.assetmanager.raiden.address self.transfermanager = transfermanager self.originating_transfer = originating_transfer hashlock = originating_transfer.lock.hashlock self.transfermanager.register_task_for_hashlock(self, hashlock) def __repr__(self): return '<{} {}>'.format( self.__class__.__name__, pex(self.address), ) def _run(self): # pylint: disable=method-hidden mediated_transfer = self.originating_transfer assetmanager = self.transfermanager.assetmanager originating_channel = assetmanager.get_channel_by_partner_address( mediated_transfer.sender) raiden = assetmanager.raiden log.debug( 'END MEDIATED TRANSFER %s -> %s msghash:%s hashlock:%s', pex(mediated_transfer.target), pex(mediated_transfer.initiator), pex(mediated_transfer.hash), pex(mediated_transfer.lock.hashlock), ) secret_request = SecretRequest(mediated_transfer.lock.hashlock) raiden.sign(secret_request) response = self.send_and_wait_valid(raiden, mediated_transfer, secret_request) if response is None: timeout_message = originating_channel.create_timeouttransfer_for( mediated_transfer) raiden.send_async(mediated_transfer.sender, timeout_message) self.transfermanager.on_hashlock_result( mediated_transfer.lock.hashlock, False) return # register the secret so that a balance proof can be created but don't # claim until our partner has informed us that it's internal state is # updated originating_channel.register_secret(response.secret) secret_message = Secret(response.secret) raiden.sign(secret_message) raiden.send_async(mediated_transfer.sender, secret_message) # wait for the secret from `sender` to claim the lock while True: response = self.response_message.wait() # critical write section self.response_message = AsyncResult() # /critical write section if isinstance( response, Secret) and response.sender == mediated_transfer.sender: originating_channel.claim_lock(response.secret) self.transfermanager.on_hashlock_result( mediated_transfer.lock.hashlock, True) return def send_and_wait_valid(self, raiden, mediated_transfer, secret_request): message_timeout = raiden.config['msg_timeout'] current_time = time.time() limit_time = current_time + message_timeout self.response_message = AsyncResult() raiden.send_async(mediated_transfer.initiator, secret_request) while current_time <= limit_time: response = self.response_message.wait(limit_time - current_time) # critical write section self.response_message = AsyncResult() # /critical write section if response is None: log.error( 'SECRETREQUEST TIMED OUT node:%s msghash:%s hashlock:%s', pex(raiden.address), pex(secret_request.hash), pex(mediated_transfer.lock.hashlock), ) return None if isinstance(response, Secret): if sha3(response.secret) != mediated_transfer.lock.hashlock: log.error('Secret doesnt match the hashlock, ignoring.') continue return response return None
def takes_too_long(self, noticear=None): if noticear is not None: noticear.set(True) ar = AsyncResult() ar.wait()
class AlarmTask(Runnable): """ Task to notify when a block is mined. """ def __init__(self, chain): super().__init__() self.callbacks = list() self.chain = chain self.chain_id = None self.last_block_number = None self._stop_event = AsyncResult() # TODO: Start with a larger sleep_time and decrease it as the # probability of a new block increases. self.sleep_time = 0.5 def _run(self): # pylint: disable=method-hidden try: self.loop_until_stop() finally: self.callbacks = list() def register_callback(self, callback): """ Register a new callback. Note: The callback will be executed in the AlarmTask context and for this reason it should not block, otherwise we can miss block changes. """ if not callable(callback): raise ValueError('callback is not a callable') self.callbacks.append(callback) def remove_callback(self, callback): """Remove callback from the list of callbacks if it exists""" if callback in self.callbacks: self.callbacks.remove(callback) def loop_until_stop(self): # The AlarmTask must have completed its first_run() before starting # the background greenlet. # # This is required because the first run will synchronize the node with # the blockchain since the last run. assert self.chain_id, 'chain_id not set' assert self.last_block_number, 'last_block_number not set' chain_id = self.chain_id sleep_time = self.sleep_time while self._stop_event.wait(sleep_time) is not True: last_block_number = self.last_block_number current_block = self.chain.block_number() if chain_id != self.chain.network_id: raise RuntimeError( 'Changing the underlying blockchain while the Raiden node is running ' 'is not supported.', ) if current_block != last_block_number: log.debug('new block', number=current_block) if current_block > last_block_number + 1: missed_blocks = current_block - last_block_number - 1 log.info( 'missed blocks', missed_blocks=missed_blocks, current_block=current_block, ) self._run_callbacks(current_block) def first_run(self): # callbacks must be executed during the first run to update the node state assert self.callbacks, 'callbacks not set' chain_id = self.chain.network_id current_block = self.chain.block_number() log.debug('starting at block number', current_block=current_block) self._run_callbacks(current_block) self.chain_id = chain_id def _run_callbacks(self, current_block): remove = list() for callback in self.callbacks: result = callback(current_block) if result is REMOVE_CALLBACK: remove.append(callback) for callback in remove: self.callbacks.remove(callback) self.last_block_number = current_block def stop(self): self._stop_event.set(True) return self.join()
class TransferTask(Task): """ Normal Operation (Transfer A > C) A: Initiator Creates Secret A: MediatedTransfer > B B: MediatedTransfer > C C: SecretRequest > A (implicitly signs, that valid transfer was received) A: Secret > C C: Secret > B Timeout (Transfer A > C) A: Initiator Creates Secret A: MediatedTransfer > B B: MediatedTransfer > C Failure: No Ack from C B: TransferTimeout > A Resolution: A won't reveal the secret, tries new transfer, B bans C CancelTransfer (Transfer A > D) A: Initiator Creates Secret A: MediatedTransfer > B B: MediatedTransfer > C Failure: C can not establish path to D (e.g. insufficient distributable, no active node) C: CancelTransfer > B (levels out balance) B: MediatedTransfer > C2 C2: MediatedTransfer > D ... """ block_expiration = 120 # FIXME, this needs to timeout on block expiration timeout_per_hop = 10 def __init__(self, transfermanager, amount, target, hashlock, expiration=None, originating_transfer=None, secret=None): # fee! assert isinstance(transfermanager, transfermanagermodule.TransferManager) self.transfermanager = transfermanager self.assetmanager = transfermanager.assetmanager self.raiden = transfermanager.assetmanager.raiden self.amount = amount self.target = target self.hashlock = hashlock self.expiration = None or 10 # fixme assert secret or originating_transfer self.originating_transfer = originating_transfer # no sender == self initiated transfer self.secret = secret super(TransferTask, self).__init__() print "INIT", self self.transfermanager.on_task_started(self) def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address)) @property def isinitiator(self): "whether this node initiated the transfer" return not self.originating_transfer def _run(self): if self.isinitiator: initiator = self.raiden.address else: initiator = self.originating_transfer.initiator # look for shortest path for path in self.assetmanager.channelgraph.get_paths(self.raiden.address, self.target): print "TRYING {} with path {}".format(self, lpex(path)) assert path[0] == self.raiden.address assert path[1] in self.assetmanager.channels assert path[-1] == self.target recipient = path[1] # check if channel is active if not self.assetmanager.channel_isactive(recipient): continue channel = self.assetmanager.channels[recipient] # check if we have enough funds ,fixme add limit per transfer if self.amount > channel.distributable: continue # calculate fee, calc expiration t = channel.create_lockedtransfer(self.amount, self.expiration, self.hashlock) t = t.to_mediatedtransfer(self.target, initiator=initiator, fee=0) # fixme fee self.raiden.sign(t) channel.register_transfer(t) # send mediated transfer msg = self.send_transfer(recipient, t, path) print "SEND RETURNED {} {}".format(self, msg) if isinstance(msg, CancelTransfer): continue # try with next path elif isinstance(msg, TransferTimeout): # stale hashlock if not self.isinitiator: self.raiden.send(self.originating_transfer.sender, msg) return self.on_completion(False) elif isinstance(msg, Secret): assert self.originating_transfer assert msg.hashlock == self.hashlock if self.originating_transfer.sender != self.originating_transfer.initiator: fwd = Secret(msg.secret) self.raiden.sign(fwd) self.raiden.send(self.originating_transfer.sender, fwd) else: print "NOT FORWARDING SECRET TO ININTIATOR" return self.on_completion(True) elif isinstance(msg, SecretRequest): assert self.isinitiator assert msg.sender == self.target msg = Secret(self.secret) self.raiden.sign(msg) self.raiden.send(self.target, msg) # apply secret to own channel channel.claim_locked(self.secret) return self.on_completion(True) # we did not find a path, send CancelTransfer if self.originating_transfer: channel = self.assetmanager.channels[self.originating_transfer.sender] t = channel.create_canceltransfer(self.originating_transfer) channel.register_transfer(t) self.raiden.sign(t) self.raiden.send(self.originating_transfer.sender, t) return self.on_completion(False) def on_event(self, msg): print "SET EVENT {} {} {}".format(self, id(self.event), msg) if self.event.ready(): print "ALREADY HAD EVENT {} {} now {}".format(self, self.event.get(), msg) assert self.event and not self.event.ready() self.event.set(msg) def send_transfer(self, recipient, transfer, path): self.event = AsyncResult() # http://www.gevent.org/gevent.event.html self.raiden.send(recipient, transfer) timeout = self.timeout_per_hop * (len(path) - 1) # fixme, consider no found paths msg = self.event.wait(timeout) print "HAVE EVENT {} {}".format(self, msg) if msg is None: # timeout print "TIMEOUT! " * 5 msg = TransferTimeout(echo=transfer.hash, hashlock=transfer.lock.hashlock) self.raiden.sign(msg) return msg if isinstance(msg, CancelTransfer): assert msg.hashlock == transfer.lock.hashlock assert msg.amount == transfer.lock.amount assert msg.recipient == transfer.sender == self.raiden.address channel = self.assetmanager.channels[msg.recipient] channel.register_transfer(msg) return msg # try with next path elif isinstance(msg, TransferTimeout): assert msg.echo == transfer.hash return msg # send back StaleHashLock, we need new hashlock elif isinstance(msg, Secret): # done exit assert msg.hashlock == self.hashlock # channel = self.assetmanager.channels[msg.recipient] # channel.claim_locked(msg.secret) # fixme this is also done by assetmanager return msg elif isinstance(msg, SecretRequest): # reveal secret print "SECRETREQUEST RECEIVED {}".format(msg) assert msg.sender == self.target return msg assert False, "Not Implemented"
class StartMediatedTransferTask(Task): def __init__(self, transfermanager, amount, target): super(StartMediatedTransferTask, self).__init__() self.amount = amount self.address = transfermanager.assetmanager.raiden.address self.target = target self.transfermanager = transfermanager def __repr__(self): return '<{} {}>'.format( self.__class__.__name__, pex(self.address), ) def _run(self): # pylint: disable=method-hidden,too-many-locals amount = self.amount target = self.target raiden = self.transfermanager.assetmanager.raiden fee = 0 # there are no guarantees that the next_hop will follow the same route routes = self.transfermanager.assetmanager.get_best_routes( amount, target, lock_timeout=None, ) for path, channel in routes: # try a new secret secret = sha3(hex(random.getrandbits(256))) hashlock = sha3(secret) self.transfermanager.register_task_for_hashlock(self, hashlock) lock_expiration = ( raiden.chain.block_number() + channel.settle_timeout - raiden.config['reveal_timeout'] ) mediated_transfer = channel.create_mediatedtransfer( raiden.address, target, fee, amount, lock_expiration, hashlock, ) raiden.sign(mediated_transfer) channel.register_transfer(mediated_transfer) response = self.send_and_wait_valid(raiden, path, mediated_transfer) # `next_hop` timedout if response is None: self.transfermanager.on_hashlock_result(hashlock, False) # someone down the line timedout / couldn't proceed elif isinstance(response, (RefundTransfer, TransferTimeout)): self.transfermanager.on_hashlock_result(hashlock, False) # `target` received the MediatedTransfer elif response.sender == target and isinstance(response, SecretRequest): secret_message = Secret(secret) raiden.sign(secret_message) raiden.send(target, secret_message) # this might cause synchronization problems, since it can take # a while for the partner to receive the secret channel.claim_locked(secret) self.transfermanager.on_hashlock_result(hashlock, True) # done, don't try a new path return else: log.error('Unexpected response {}'.format(repr(response))) self.transfermanager.on_hashlock_result(hashlock, False) def send_and_wait_valid(self, raiden, path, mediated_transfer): # pylint: disable=no-self-use """ Send the `mediated_transfer` and wait for either a message from `target` or the `next_hop`. Validate the message received and discards the invalid ones. The most important case being next_hop sending a SecretRequest. """ message_timeout = raiden.config['msg_timeout'] next_hop = path[1] target = path[-1] transfer_details = 'path:{} hash:{}'.format( lpex(path), pex(mediated_transfer.hash), ) log.debug('MEDIATED TRANSFER STARTED {}'.format(transfer_details)) current_time = time.time() limit_time = current_time + message_timeout self.event = AsyncResult() raiden.send(next_hop, mediated_transfer) while current_time <= limit_time: response = self.event.wait(limit_time - current_time) # critical write section self.event = AsyncResult() # reset so that a new value can be received # /critical write section current_time = time.time() if response is None: log.debug('MEDIATED TRANSFER TIMED OUT {}'.format(transfer_details)) return None if response.sender == next_hop: if isinstance(response, (RefundTransfer, TransferTimeout)): return response else: log.info('Partner {} sent an invalid message'.format(pex(next_hop))) return None if response.sender == target: if isinstance(response, SecretRequest): return response else: log.info('target {} sent an invalid message'.format(pex(target))) return None log.error('Invalid message ignoring. {}'.format(repr(response))) return None
class AlarmTask(Task): """ Task to notify when a block is mined. """ def __init__(self, chain): super(AlarmTask, self).__init__() self.callbacks = list() self.stop_event = AsyncResult() self.wait_time = 0.5 self.chain = chain self.last_block_number = self.chain.block_number() def register_callback(self, callback): """ Register a new callback. Note: This callback will be executed in the AlarmTask context and for this reason it should not block, otherwise we can miss block changes. """ if not callable(callback): raise ValueError('callback is not a callable') self.callbacks.append(callback) def _run(self): # pylint: disable=method-hidden stop = None result = None last_loop = time.time() log.debug('starting block number', block_number=self.last_block_number) while stop is None: current_block = self.chain.block_number() if current_block > self.last_block_number + 1: difference = current_block - self.last_block_number - 1 log.error( 'alarm missed %s blocks', difference, ) if current_block != self.last_block_number: self.last_block_number = current_block log.debug('new block', number=current_block, timestamp=last_loop) remove = list() for callback in self.callbacks: try: result = callback(current_block) except: log.exception('unexpected exception on alarm') else: if result is REMOVE_CALLBACK: remove.append(callback) for callback in remove: self.callbacks.remove(callback) # we want this task to iterate in the tick of `wait_time`, so take # into account how long we spent executing one tick. work_time = time.time() - last_loop if work_time > self.wait_time: log.warning( 'alarm loop is taking longer than the wait time', work_time=work_time, wait_time=self.wait_time, ) sleep_time = 0.001 else: sleep_time = self.wait_time - work_time stop = self.stop_event.wait(sleep_time) last_loop = time.time() def stop(self): self.stop_event.set(True)
class AlarmTask(gevent.Greenlet): """ Task to notify when a block is mined. """ def __init__(self, chain): super().__init__() # TODO: Start with a larger sleep_time and decrease it as the # probability of a new block increases. sleep_time = 0.5 self.callbacks = list() self.chain = chain self.chain_id = None self.last_block_number = None self.stop_event = AsyncResult() self.sleep_time = sleep_time def _run(self): # pylint: disable=method-hidden try: self.loop_until_stop() except RaidenShuttingDown: pass finally: self.callbacks = list() def register_callback(self, callback): """ Register a new callback. Note: The callback will be executed in the AlarmTask context and for this reason it should not block, otherwise we can miss block changes. """ if not callable(callback): raise ValueError('callback is not a callable') self.callbacks.append(callback) def remove_callback(self, callback): """Remove callback from the list of callbacks if it exists""" if callback in self.callbacks: self.callbacks.remove(callback) def loop_until_stop(self): # The AlarmTask must have completed its first_run() before starting # the background greenlet. # # This is required because the first run will synchronize the node with # the blockchain since the last run. assert self.chain_id, 'chain_id not set' assert self.last_block_number, 'last_block_number not set' chain_id = self.chain_id sleep_time = self.sleep_time while self.stop_event.wait(sleep_time) is not True: last_block_number = self.last_block_number current_block = self.chain.block_number() if chain_id != self.chain.network_id: raise RuntimeError( 'Changing the underlying blockchain while the Raiden node is running ' 'is not supported.', ) if current_block != last_block_number: log.debug('new block', number=current_block) if current_block > last_block_number + 1: missed_blocks = current_block - last_block_number - 1 log.error( 'missed blocks', missed_blocks=missed_blocks, current_block=current_block, ) self.run_callbacks(current_block, chain_id) def first_run(self): # callbacks must be executed during the first run to update the node state assert self.callbacks, 'callbacks not set' chain_id = self.chain.network_id current_block = self.chain.block_number() log.debug('starting at block number', current_block=current_block) self.run_callbacks(current_block, chain_id) self.chain_id = chain_id def run_callbacks(self, current_block, chain_id): remove = list() for callback in self.callbacks: result = callback(current_block, chain_id) if result is REMOVE_CALLBACK: remove.append(callback) for callback in remove: self.callbacks.remove(callback) self.last_block_number = current_block def stop_async(self): self.stop_event.set(True)
class MediateTransferTask(Task): # pylint: disable=too-many-instance-attributes def __init__(self, transfermanager, originating_transfer, fee): super(MediateTransferTask, self).__init__() self.address = transfermanager.assetmanager.raiden.address self.transfermanager = transfermanager self.fee = fee self.originating_transfer = originating_transfer hashlock = originating_transfer.lock.hashlock self.transfermanager.register_task_for_hashlock(self, hashlock) def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, pex(self.address)) def _run(self): # pylint: disable=method-hidden,too-many-locals,too-many-branches,too-many-statements fee = self.fee transfer = self.originating_transfer assetmanager = self.transfermanager.assetmanager raiden = assetmanager.raiden originating_channel = assetmanager.partneraddress_channel[ transfer.sender] assetmanager.register_channel_for_hashlock( originating_channel, transfer.lock.hashlock, ) lock_expiration = transfer.lock.expiration - raiden.config[ 'reveal_timeout'] lock_timeout = lock_expiration - raiden.chain.block_number() # there are no guarantees that the next_hop will follow the same route routes = assetmanager.get_best_routes( transfer.lock.amount, transfer.target, lock_timeout, ) if log.isEnabledFor(logging.DEBUG): log.debug( 'MEDIATED TRANSFER initiator:%s node:%s target:%s', pex(transfer.initiator), pex(self.address), pex(transfer.target), ) for path, forward_channel in routes: next_hop = path[1] mediated_transfer = forward_channel.create_mediatedtransfer( transfer.initiator, transfer.target, fee, transfer.lock.amount, lock_expiration, transfer.lock.hashlock, ) raiden.sign(mediated_transfer) if log.isEnabledFor(logging.DEBUG): log.debug( 'MEDIATED TRANSFER NEW PATH path:%s hashlock:%s', lpex(path), pex(transfer.lock.hashlock), ) # Using assetmanager to register the interest because it outlives # this task, the secret handling will happend only _once_ assetmanager.register_channel_for_hashlock( forward_channel, transfer.lock.hashlock, ) forward_channel.register_transfer(mediated_transfer) response = self.send_and_wait_valid(raiden, path, mediated_transfer) if response is None: timeout_message = forward_channel.create_timeouttransfer_for( transfer) raiden.send_async(transfer.sender, timeout_message) self.transfermanager.on_hashlock_result( transfer.lock.hashlock, False) return if isinstance(response, RefundTransfer): if response.lock.amount != transfer.amount: log.info( 'Partner %s sent an refund message with an invalid amount', pex(next_hop), ) timeout_message = forward_channel.create_timeouttransfer_for( transfer) raiden.send_async(transfer.sender, timeout_message) self.transfermanager.on_hashlock_result( transfer.lock.hashlock, False) return else: forward_channel.register_transfer(response) elif isinstance(response, Secret): # update all channels and propagate the secret (this doesnt claim the lock yet) assetmanager.handle_secret(response.secret) # wait for the secret from `sender` while True: response = self.response_message.wait() # critical write section self.response_message = AsyncResult() # /critical write section # NOTE: this relies on the fact RaindenService dispatches # messages based on the `hashlock` calculated from the # secret, so we know this `response` message secret matches # the secret from the `next_hop` if isinstance( response, Secret) and response.sender == transfer.sender: originating_channel.claim_lock(response.secret) self.transfermanager.on_hashlock_result( transfer.lock.hashlock, True) return # No suitable path avaiable (e.g. insufficient distributable, no active node) # Send RefundTransfer to the originating node, this has the effect of # backtracking in the graph search of the raiden network. from_address = transfer.sender from_channel = assetmanager.partneraddress_channel[from_address] refund_transfer = from_channel.create_refundtransfer_for(transfer) from_channel.register_transfer(refund_transfer) raiden.sign(refund_transfer) raiden.send_async(from_address, refund_transfer) log.debug( 'REFUND MEDIATED TRANSFER from=%s node:%s hashlock:%s', pex(from_address), pex(raiden.address), pex(transfer.lock.hashlock), ) self.transfermanager.on_hashlock_result(transfer.lock.hashlock, False) return def send_and_wait_valid(self, raiden, path, mediated_transfer): message_timeout = raiden.config['msg_timeout'] next_hop = path[1] current_time = time.time() limit_time = current_time + message_timeout self.response_message = AsyncResult() raiden.send_async(next_hop, mediated_transfer) while current_time <= limit_time: response = self.response_message.wait(limit_time - current_time) # critical write section self.response_message = AsyncResult( ) # reset so that a new value can be received # /critical write section current_time = time.time() if response is None: log.error( 'MEDIATED TRANSFER TIMED OUT node:%s timeout:%s msghash:%s hashlock:%s', pex(raiden.address), message_timeout, pex(mediated_transfer.hash), pex(mediated_transfer.lock.hashlock), ) return None if isinstance(response, Secret): if sha3(response.secret) != mediated_transfer.lock.hashlock: log.error('Secret doesnt match the hashlock, ignoring.') continue return response if response.target != raiden.address or response.sender != next_hop: log.error('Invalid message supplied to the task. %s', repr(response)) continue if isinstance(response, RefundTransfer): return response log.error('Partner sent an invalid message. %s', repr(response)) return None
class MediateTransferTask(Task): # pylint: disable=too-many-instance-attributes def __init__(self, transfermanager, originating_transfer, fee): super(MediateTransferTask, self).__init__() self.address = transfermanager.assetmanager.raiden.address self.transfermanager = transfermanager self.fee = fee self.originating_transfer = originating_transfer hashlock = originating_transfer.lock.hashlock self.transfermanager.register_task_for_hashlock(self, hashlock) def __repr__(self): return '<{} {}>'.format( self.__class__.__name__, pex(self.address) ) def _run(self): # pylint: disable=method-hidden,too-many-locals,too-many-branches,too-many-statements fee = self.fee transfer = self.originating_transfer assetmanager = self.transfermanager.assetmanager raiden = assetmanager.raiden originating_channel = assetmanager.partneraddress_channel[transfer.sender] assetmanager.register_channel_for_hashlock( originating_channel, transfer.lock.hashlock, ) lock_expiration = transfer.lock.expiration - raiden.config['reveal_timeout'] lock_timeout = lock_expiration - raiden.chain.block_number() # there are no guarantees that the next_hop will follow the same route routes = assetmanager.get_best_routes( transfer.lock.amount, transfer.target, lock_timeout, ) for path, channel in routes: next_hop = path[1] mediated_transfer = channel.create_mediatedtransfer( transfer.initiator, transfer.target, fee, transfer.lock.amount, lock_expiration, transfer.lock.hashlock, ) raiden.sign(mediated_transfer) assetmanager.register_channel_for_hashlock( channel, transfer.lock.hashlock, ) channel.register_transfer(mediated_transfer) response = self.send_and_wait_valid(raiden, path, mediated_transfer) if response is None: timeout = channel.create_timeouttransfer_for(transfer) raiden.send(transfer.sender, timeout) self.transfermanager.on_hashlock_result(transfer.hashlock, False) return if isinstance(response, RefundTransfer): if response.lock.amount != transfer.amount: log.info('Partner {} sent an refund message with an invalid amount'.format( pex(next_hop), )) timeout = channel.create_timeouttransfer_for(transfer) raiden.send(transfer.sender, timeout) self.transfermanager.on_hashlock_result(transfer.hashlock, False) return else: channel.register_transfer(response) elif isinstance(response, Secret): # update all channels and propagate the secret assetmanager.register_secret(response.secret) self.transfermanager.on_hashlock_result(transfer.lock.hashlock, True) return # No suitable path avaiable (e.g. insufficient distributable, no active node) # Send RefundTransfer to the originating node, this has the effect of # backtracking in the graph search of the raiden network. from_address = transfer.sender from_channel = assetmanager.partneraddress_channel[from_address] refund_transfer = from_channel.create_refundtransfer_for(transfer) from_channel.register_transfer(refund_transfer) raiden.sign(refund_transfer) raiden.send(from_address, refund_transfer) log.debug('REFUND MEDIATED TRANSFER from={} {}'.format( pex(from_address), pex(raiden.address), )) self.transfermanager.on_hashlock_result(transfer.hashlock, False) return def send_and_wait_valid(self, raiden, path, mediated_transfer): message_timeout = raiden.config['msg_timeout'] next_hop = path[1] transfer_details = 'path:{} hash:{} initiator:{}'.format( lpex(path), pex(mediated_transfer.hash), pex(mediated_transfer.initiator), ) log.debug('MEDIATED TRANSFER {}'.format(transfer_details)) current_time = time.time() limit_time = current_time + message_timeout self.event = AsyncResult() raiden.send(next_hop, mediated_transfer) while current_time <= limit_time: response = self.event.wait(limit_time - current_time) # critical write section self.event = AsyncResult() # reset so that a new value can be received # /critical write section current_time = time.time() if response is None: log.error('MEDIATED TRANSFER TIMED OUT {} timeout:{}'.format( transfer_details, message_timeout, )) return None if isinstance(response, Secret): if response.hashlock != mediated_transfer.lock.hashlock: log.error('Secret doesnt match the hashlock, ignoring.') continue return response if response.target != raiden.address or response.sender != next_hop: log.error('Invalid message supplied to the task. {}'.format(repr(response))) continue if isinstance(response, RefundTransfer): return response log.error('Partner sent an invalid message. {}'.format(repr(response))) return None
class LogListenerTask(Task): """ Task for polling for filter changes. """ def __init__(self, listener_name, filter_, callback, contract_translator, events_poll_timeout=DEFAULT_EVENTS_POLL_TIMEOUT): """ Args: listener_name (str): A name to distinguish listener tasks. filter_ (raiden.network.rpc.client.Filter): A proxy for calling the blockchain's filter api. callback (function): A function to be called once an event happens. contract_translator (ethereum.abi.ContractTranslator): A contract translator to decode the event data. events_poll_timeout (float): How long the tasks should sleep before polling again. """ super(LogListenerTask, self).__init__() self.listener_name = listener_name self.filter_ = filter_ self.callback = callback self.contract_translator = contract_translator self.stop_event = AsyncResult() self.sleep_time = events_poll_timeout # exposes the AsyncResult timer, this allows us to raise the timeout # inside this Task to force an update: # # task.kill(task.timeout) # self.timeout = None def __repr__(self): return '<LogListenerTask {}>'.format(self.listener_name) def _run(self): # pylint: disable=method-hidden stop = None while stop is None: filter_changes = self.filter_.changes() for log_event in filter_changes: log.debug('New Events', task=self.listener_name) event = self.contract_translator.decode_event( log_event['topics'], log_event['data'], ) if event is not None: originating_contract = log_event['address'] try: self.callback(originating_contract, event) except: log.exception('unexpected exception on log listener') self.timeout = Timeout( self.sleep_time) # wait() will call cancel() stop = self.stop_event.wait(self.timeout) def stop(self): self.stop_event.set(True)
class ClientService(object): def __init__(self, _websocket, user_id, _config): self._user_id = user_id self._config = _config self._websocket = _websocket assert isinstance(self._websocket, WebSocket) matches = _config.get(ConfigField.MATCH) # 订阅的条件, 目前是regex的字符串列表 fields = _config.get(ConfigField.FIELD) # 返回的订阅数据字段列表 self.compress_type = CompressValues.DEFAULT \ if ConfigField.COMPRESS in _config and _config[ConfigField.COMPRESS] else None self._patterns = [] for item in matches: if isinstance(item, basestring): self._patterns.append(re.compile(item)) if item not in self.all_patterns: self.all_patterns[item] = {self._user_id} else: self.all_patterns[item].add(self._user_id) self._fields = set([item for item in fields if item in ALL_FIELDS]) # 存在file_id表明数据被持久化储存 self._fields.add(FILE_ID) self._publish = AsyncResult() self.clients[user_id] = self def get_publish(self): result = self._publish.wait() self._publish = AsyncResult() return result fields = property(lambda o: o._fields) publish = property(get_publish, lambda o, v: o._publish.set(v)) clients = {} all_patterns = {} def close(self): for k, v in self.all_patterns.items(): assert isinstance(v, set) if self._user_id in v: v.remove(self._user_id) if len(v) == 0: self.all_patterns.pop(k) self.clients.pop(self._user_id, None) def get(self): # block return self._publish.get() def match(self, url): for p in self._patterns: if p.match(url): return True return False @classmethod def add(cls, **kwargs): # 找到所有需要广播的用户 url = kwargs.get("url") all_user_id_set = set() for pattern_str, user_id_set in cls.all_patterns.iteritems(): assert isinstance(user_id_set, set) # 使用系统自带的正则缓存 p_obj = re.compile(pattern_str) if p_obj.match(url): all_user_id_set.update(user_id_set) # 数据格式化 if FILE_ID in kwargs: kwargs[FILE_ID] = str(kwargs[FILE_ID]) for name in kwargs.keys(): if name in BASE64_FIELDS: # 编码成base64 方便转换成json kwargs[name] = kwargs[name].encode("base64") elif name in DICT_FIELDS: # 将请求头和返回头转换成dict kwargs[name] = dict(kwargs[name]) # 广播 for user_id in all_user_id_set: obj = cls.clients[user_id] obj.publish = { k: v for k, v in kwargs.iteritems() if k in obj.fields } def one_send(self): # 阻塞 result = self.publish again_send = {} for k, v in result.items(): if k in BIG_DATA_FIELDS: again_send[k] = result.pop(k) again_data = "" if len(again_send) > 0: again_data = json.dumps(again_send) if self.compress_type == CompressValues.DEFAULT: again_data = zip_compress(again_data) result[AGAIN_LENGTH] = len(again_data) self._websocket.send(json.dumps(result)) if len(again_send) > 0: sio = StringIO(again_data) send_time = len(again_data) / (ONE_SEND_LENGTH * 1.0) for _ in xrange( int(send_time) + 1 if send_time > int(send_time) else int(send_time)): self._websocket.send(sio.read(ONE_SEND_LENGTH), True)
class AlarmTask(Task): """ Task to notify when a block is mined. """ def __init__(self, chain): super(AlarmTask, self).__init__() self.callbacks = list() self.stop_event = AsyncResult() self.chain = chain self.last_block_number = None # TODO: Start with a larger wait_time and decrease it as the # probability of a new block increases. self.wait_time = 0.5 self.last_loop = time.time() def register_callback(self, callback): """ Register a new callback. Note: The callback will be executed in the AlarmTask context and for this reason it should not block, otherwise we can miss block changes. """ if not callable(callback): raise ValueError('callback is not a callable') self.callbacks.append(callback) def remove_callback(self, callback): """Remove callback from the list of callbacks if it exists""" if callback in self.callbacks: self.callbacks.remove(callback) def _run(self): # pylint: disable=method-hidden log.debug('starting block number', block_number=self.last_block_number) sleep_time = 0 while self.stop_event.wait(sleep_time) is not True: self.poll_for_new_block() # we want this task to iterate in the tick of `wait_time`, so take # into account how long we spent executing one tick. self.last_loop = time.time() work_time = self.last_loop - self.last_loop if work_time > self.wait_time: log.warning( 'alarm loop is taking longer than the wait time', work_time=work_time, wait_time=self.wait_time, ) sleep_time = 0.001 else: sleep_time = self.wait_time - work_time # stopping self.callbacks = list() def poll_for_new_block(self): current_block = self.chain.block_number() if current_block > self.last_block_number + 1: difference = current_block - self.last_block_number - 1 log.error( 'alarm missed %s blocks', difference, ) if current_block != self.last_block_number: log.debug( 'new block', number=current_block, timestamp=self.last_loop, ) self.last_block_number = current_block remove = list() for callback in self.callbacks: try: result = callback(current_block) except: # pylint: disable=bare-except log.exception('unexpected exception on alarm') else: if result is REMOVE_CALLBACK: remove.append(callback) for callback in remove: self.callbacks.remove(callback) def start(self): self.last_block_number = self.chain.block_number() super(AlarmTask, self).start() def stop_and_wait(self): self.stop_event.set(True) gevent.wait(self) def stop_async(self): self.stop_event.set(True)
class DolphinConnection(object): def __init__(self, host="localhost", port=6000): ''' Creating a new DolphinConnection instance, pointing to the DolphinConnection Server specified by host and port. The connection must be established explicitly with connect(). host and port can be overwritten, followed by another connect() call to reconnect. ''' self.host = host self.port = port self._connected = False self._sock = None self._cFunc = None self._dcFunc = None self._callbacks = {} self._buf = "" self._sep = "\n" self._feedback = AsyncResult() self._feedback.set(None) def isConnected(self): ''' Returns whether the DolphinConnection instance is connected to the corresponding server defined by host and port. ''' return self._connected def connect(self): ''' Tries to establish a connection to the server. If it succeeds, the onConnect callback will be called. If it fails, the onDisconnect callback will be called. ''' self.disconnect() self._connected = True try: self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._sock.connect((self.host, self.port)) logger.info("DolphinConnection connection to %s:%d established! " + "Ready for work!", self.host, self.port) gevent.spawn(self._recv) if self._cFunc: self._cFunc(self) except socket.error: logger.info("DolphinConnection connection to %s:%d failed.", self.host, self.port) self._disconnect(DisconnectReason.CONNECTION_FAILED) def disconnect(self): ''' Disconnects the socket from the server. The onDisconnect callback will be called with CONNECTION_CLOSED_BY_HOST ''' self._disconnect(DisconnectReason.CONNECTION_CLOSED_BY_HOST) def _disconnect(self, reason): if not self._connected: return self._connected = False self._feedback.set(False) try: self._sock.close() except: pass if self._dcFunc: self._dcFunc(self, reason) def onConnect(self, func): ''' Sets the callback that will be called after a connection has been successfully established. Callback is initially None, and can again be assigned to None. ''' if not hasattr(func, '__call__'): raise ValueError("onDisconnect callback must be callable.") self._cFunc = func def onDisconnect(self, func): ''' Sets the callback that will be called after a connection attempt fails, an active connection gets closed or the connection gets lost. A DisconnectReason enum will be the parameter. Callback is initially None, and can again be assigned to None. ''' if not hasattr(func, '__call__'): raise ValueError("onDisconnect callback must be callable.") self._dcFunc = func def startBatch(self): ''' Call this function to send following commands in a batch. All following commands are guaranteed to be executed at once in Dolphin. Is done by buffering and not executing anything until endBatch() is called. ''' self._sep = ";" def endBatch(self): ''' Ends the batch started with startBatch(). All buffered commands gets executed now and no more buffering is done. ''' self._sep = "\n" self._cmd("") def volume(self, v): ''' Sets Dolphin's Audio. :param v: 0-100, audio level ''' self._cmd("VOLUME %d" % v) def write(self, mode, addr, val): ''' Sends a command to write <mode> bytes of data to the given address. <mode> must be 8, 16 or 32. ''' self._cmd("WRITE %d %d %d" % (mode, addr, val)) def writeMulti(self, addr, vals): ''' Sends a command to write the bytes <vals>, starting at address <addr>. ''' self._cmd("WRITE_MULTI %d %s" % (addr, " ".join(str(v) for v in vals))) def read(self, mode, addr, callback): ''' Sends a command to send back <mode> bytes of data at the given address. The given callback function gets called with the returned value as parameter. <mode> must be 8, 16 or 32. ''' self._reg_callback(addr, callback, False) self._cmd("READ %d %d" % (mode, addr)) def _subscribe(self, mode, addr, callback): ''' Sends a command to send back <mode> bytes of data at the given address, repeating each time the value changes. The given callback function gets called with the returned value as parameter. <mode> must be 8, 16 or 32. ''' self._reg_callback(addr, callback, True) self._cmd("SUBSCRIBE %d %d" % (mode, addr)) def _subscribeMulti(self, size, addr, callback): ''' Sends a command to send back <size> bytes of data starting at the given address, repeating each time the value changes. Useful for strings and arrays. The given callback function gets called with the returned values in a list as parameter. ''' self._reg_callback(addr, callback, True) self._cmd("SUBSCRIBE_MULTI %d %d" % (size, addr)) def write8(self, addr, val): ''' Sends a command to write 8 bytes of data to the given address. ''' self.write(8, addr, val) def write16(self, addr, val): ''' Sends a command to write 16 bytes of data to the given address. ''' self.write(16, addr, val) def write32(self, addr, val): ''' Sends a command to write 32 bytes of data to the given address. ''' self.write(32, addr, val) def read8(self, addr, callback): ''' Sends a command to send back 8 bytes of data at the given address. The given callback function gets called with the returned value as parameter. ''' self.read(8, addr, callback) def read16(self, addr, callback): ''' Sends a command to send back 16 bytes of data at the given address. The given callback function gets called with the returned value as parameter. ''' self.read(16, addr, callback) def read32(self, addr, callback): ''' Sends a command to send back 32 bytes of data at the given address. The given callback function gets called with the returned value as parameter. ''' if addr % 4 != 0: raise ValueError("Read32 address must be whole word; " + "multiple of 4") self.read(32, addr, callback) def subscribe8(self, addr, callback): ''' Sends a command to send back 8 bytes of data at the given address, repeating each time the value changes. The given callback function gets called with the returned value as parameter. ''' self._subscribe(8, addr, callback) def subscribe16(self, addr, callback): ''' Sends a command to send back 16 bytes of data at the given address, repeating each time the value changes. The given callback function gets called with the returned value as parameter. ''' self._subscribe(16, addr, callback) def subscribe32(self, addr, callback): ''' Sends a command to send back 32 bytes of data at the given address, repeating each time the value changes. The given callback function gets called with the returned value as parameter. ''' if addr % 4 != 0: raise ValueError("Read address must be whole word; " + "multiple of 4") self._subscribe(32, addr, callback) def subscribeMulti(self, size, addr, callback): ''' Sends a command to send back <size> bytes of data starting at the given address, repeating each time any value changes. Useful for strings or arrays. The given callback function gets called with the returned values in a list as parameter. ''' self._subscribeMulti(size, addr, callback) def wiiButton(self, wiimoteIndex, buttonstates): ''' Sends 16 bit of data representing some buttonstates of the Wiimote. NOTE: The real or emulated wiimote dolphin uses gets hijacked for only roughly half a second. After this time that wiimote handled by dolphin starts to send it's buttonstates again. :param wiimoteIndex: 0-3, index of the wiimote to emulate. :param buttonstates: bitmask of the buttonstates, see http://wiibrew.org/wiki/Wiimote#Buttons for more info ''' self._cmd("BUTTONSTATES_WII %d %d" % (wiimoteIndex, buttonstates)) def gcButton(self, gcpadIndex, buttonstates, stickX=0.0, stickY=0.0, substickX=0.0, substickY=0.0): ''' Sends 16 bit of data and 2 floats representing some buttonstates of the GCPad. NOTE: The real or emulated gcpad dolphin uses gets hijacked for only roughly half a second. After this time that gcpad handled by dolphin starts to send it's buttonstates again. :param gcpadIndex: 0-3, index of the gcpad to emulate. :param buttonstates: bitmask of the buttonstates, see http://pastebin.com/raw.php?i=4txWae07 for more info :param stickX: between -1.0 and 1.0, x-position of the main stick, 0 is neutral :param stickY: between -1.0 and 1.0, y-position of the main stick, 0 is neutral :param substickX: between -1.0 and 1.0, x-position of the c-stick, 0 is neutral :param substickY: between -1.0 and 1.0, y-position of the c-stick, 0 is neutral ''' self._cmd("BUTTONSTATES_GC %d %d %f %f %f %f" % (gcpadIndex, buttonstates, stickX, stickY, substickX, substickY)) def pause(self): ''' Tells Dolphin to pause the current emulation. Resume with resume() ''' self._cmd("PAUSE") def resume(self): ''' Tells Dolphin to resume the current emulation. ''' self._cmd("RESUME") def reset(self): ''' Tells Dolphin to push the reset button. ''' self._cmd("RESET") def save(self, filename): ''' Tells Dolphin to make a savestate and save it to <filename>. ''' if any(c in filename for c in "?\"<>|"): raise ValueError("filename must not contain any of the " + "following: :?\"<> | ") self._cmd("SAVE %s" % filename) def load(self, filename): ''' Tells Dolphin to load the savestate located at <filename>. This function will block until feedback as arrived and will then return true if it succeded, else false. CAUTION: Will permanently block if dolphin was paused :( ''' if any(c in filename for c in "?\"<>|"): raise ValueError("filename must not contain any of the " + "following: ?\"<> | ") return self._cmd("LOAD %s" % filename, True) def stop(self): ''' Stops the current emulation. DolphinWatch does NOT support starting a new game then. To change the game, use insert() to insert a new iso and then reset(). ''' self._cmd("STOP") def insert(self, filename): ''' Inserts up a new game (iso). :param filename: The file (iso e.g.) to be loaded. Relative do dolphin. CAUTION: Running games can crash if the iso changes while running. To change a game, pause, then insert, and after a bit reset the game. ''' if any(c in filename for c in "?\"<>|"): raise ValueError("filename must not contain any of the " + "following: ?\"<> | ") self._cmd("INSERT %s" % filename) ###################################### # private methods below def _cmd(self, cmd, feedback=False): if not self._connected: raise socket.error("DolphinConnection is not connected and " + "therefore cannot perform actions!") if feedback: try: self._feedback.wait(1.0) except gevent.Timeout: pass # TODO got locked up :( self._feedback = AsyncResult() self._sock.send((cmd + self._sep).encode()) r = self._feedback.get(True) return r else: self._sock.send((cmd + self._sep).encode()) return True def _reg_callback(self, addr, func, _subscribe=False): self._callbacks[addr] = (func, _subscribe) def _dereg_callback(self, addr): self._callbacks.pop(addr) def _process(self, line): parts = line.split(" ") if parts[0] == "MEM": addr = int(parts[1]) val = int(parts[2]) callback = self._callbacks.get(addr) if callback: if not callback[1]: self._dereg_callback(addr) gevent.spawn(callback[0], val) else: logger.warning("No recipient for address 0x%x, value 0x%x", addr, val) elif parts[0] == "MEM_MULTI": addr = int(parts[1]) data = [int(v) for v in parts[2:]] callback = self._callbacks.get(addr) if callback: if not callback[1]: self._dereg_callback(addr) gevent.spawn(callback[0], data) else: logger.warning("No recipient for address 0x%x, data %s", addr, data) elif parts[0] == "FAIL": self._feedback.set(False) elif parts[0] == "SUCCESS": self._feedback.set(True) elif parts[0] == "LOG": level = _log_translation[int(parts[1])] dolphin_logger.log(level, " ".join(parts[2:])) else: logger.warning("Unknown incoming DolphinWatch command: %s", line) def _recv(self): while self._connected: try: data = self._sock.recv(1024) if not data: logger.info("DolphinConnection connection closed") self._disconnect(DisconnectReason.CONNECTION_CLOSED_BY_PEER) return self._buf += data.decode() except socket.error: logger.warning("DolphinConnection connection lost") self._disconnect(DisconnectReason.CONNECTION_LOST) return *lines, rest = self._buf.split("\n") self._buf = rest for line in lines: self._process(line.strip())