def test_heartbeat_current_op_over_limit(self):
self.patch_cfg('pyon.ion.process.CFG', {'container':{'timeout':{'heartbeat_proc_count_threshold':2}}})
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target
def fake_op(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
self.addCleanup(listeninev.set) # allow graceful termination
self.addCleanup(p.stop)
ar = p._routing_call(fake_op, None, listenoutev, listeninev)
listenoutev.wait(timeout=5) # wait for ctrl thread to run our op
# make sure it's over the threshold
for x in xrange(3):
hb = p.heartbeat()
self.assertEquals((True, True, False), hb)
def test_heartbeat_listener_dead(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
listeninev.set() # stop the listen loop
p.thread_manager.children[1].join(timeout=5) # wait for listen loop to terminate
hb = p.heartbeat()
self.assertEquals((False, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def test_heartbeat_current_op_over_limit(self):
self.patch_cfg(
'pyon.ion.process.CFG',
{'cc': {
'timeout': {
'heartbeat_proc_count_threshold': 2
}
}})
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target
def fake_op(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
self.addCleanup(listeninev.set) # allow graceful termination
self.addCleanup(p.stop)
ar = p._routing_call(fake_op, None, listenoutev, listeninev)
listenoutev.wait(timeout=5) # wait for ctrl thread to run our op
# make sure it's over the threshold
for x in xrange(3):
hb = p.heartbeat()
self.assertEquals((True, True, False), hb)
def test_heartbeat_with_current_op_multiple_times(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target
def fake_op(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
self.addCleanup(listeninev.set) # allow graceful termination
self.addCleanup(p.stop)
ar = p._routing_call(fake_op, None, listenoutev, listeninev)
listenoutev.wait(timeout=5) # wait for ctrl thread to run our op
for x in xrange(5):
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(5, p._heartbeat_count)
self.assertEquals(ar, p._heartbeat_op)
def mediated_transfer(initiator_app, target_app, asset, amount, identifier=None):
""" Nice to read shortcut to make a MediatedTransfer.
The secret will be revealed and the apps will be synchronized.
"""
# pylint: disable=too-many-arguments
assetmanager = initiator_app.raiden.managers_by_asset_address[asset]
has_channel = initiator_app.raiden.address in assetmanager.partneraddress_channel
# api.transfer() would do a DirectTransfer
if has_channel:
transfermanager = assetmanager.transfermanager
# Explicitly call the default identifier creation since this mock
# function here completely skips the `transfer_async()` call.
if not identifier:
identifier = transfermanager.create_default_identifier(target_app.raiden.address)
result = AsyncResult()
task = StartMediatedTransferTask(
transfermanager,
amount,
identifier,
target_app.raiden.address,
result,
)
task.start()
result.wait()
else:
initiator_app.raiden.api.transfer(
asset,
amount,
target_app.raiden.address,
identifier
)
def mediated_transfer(initiator_app, target_app, token, amount, identifier=None):
""" Nice to read shortcut to make a MediatedTransfer.
The secret will be revealed and the apps will be synchronized.
"""
# pylint: disable=too-many-arguments
tokenmanager = initiator_app.raiden.managers_by_token_address[token]
has_channel = initiator_app.raiden.address in tokenmanager.partneraddress_channel
# api.transfer() would do a DirectTransfer
if has_channel:
transfermanager = tokenmanager.transfermanager
# Explicitly call the default identifier creation since this mock
# function here completely skips the `transfer_async()` call.
if not identifier:
identifier = transfermanager.create_default_identifier(target_app.raiden.address)
result = AsyncResult()
task = StartMediatedTransferTask(
transfermanager,
amount,
identifier,
target_app.raiden.address,
result,
)
task.start()
result.wait()
else:
initiator_app.raiden.api.transfer(
token,
amount,
target_app.raiden.address,
identifier
)
def test_heartbeat_with_current_op_multiple_times(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target
def fake_op(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
self.addCleanup(listeninev.set) # allow graceful termination
self.addCleanup(p.stop)
ar = p._routing_call(fake_op, None, listenoutev, listeninev)
listenoutev.wait(timeout=5) # wait for ctrl thread to run our op
for x in xrange(5):
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(5, p._heartbeat_count)
self.assertEquals(ar, p._heartbeat_op)
def test_heartbeat_with_listeners(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
mocklistener.listen = lambda *a, **kw: fake_listen(listenoutev, listeninev)
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
# now test heartbeat!
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def send_request(self, host, request_id, entity_type, entity_id, method,
*args, **kwargs):
self.connect()
request = RpcRequest()
if host is not None:
request.host = host
else:
request.host = get_server_unique_id()
if request_id is not None:
request.request_id = request_id
request.entity_type = entity_type
request.entity_id = entity_id
request.method = method
request.args = args
request.kwargs = kwargs
self._conn.send_message(request)
future = AsyncResult()
add_future(request.request_id, future)
future.wait(5)
resp: RpcResponse = future.get()
return resp.response
def _event():
# 协程通信
e = Event()
ae = AsyncResult()
e.wait() # 唤醒
e.set() # 唤醒
ae.wait() # 唤醒
ae.set('') # 唤醒,可传递信息
def aimbot():
dw = dolphinWatch.DolphinConnection()
connected_event = AsyncResult()
dw.onConnect(connected_event.set)
dw.connect()
connected_event.wait()
while True:
time.sleep(0.1)
dw.write32(pointer + 0x6405e0, 1)
dw.save("G:/bla.sav")
def test_foo(self):
router = QueueRouter()
resultSlot1 = AsyncResult()
resultSlot2 = AsyncResult()
gevent.spawn(subscriber1, router, resultSlot1)
gevent.spawn(subscriber2, router, resultSlot2)
gevent.sleep(0) # yield to subscribers for setup
gevent.spawn(publisher, router)
result1 = resultSlot1.wait()
result2 = resultSlot2.wait()
expected = [('foobar', 0),
('foobar', 1),
('foobar', 2)]
self.assertEqual(result1, expected)
self.assertEqual(result2, expected)
def test_foo(self):
router = QueueRouter()
resultSlot1 = AsyncResult()
resultSlot2 = AsyncResult()
gevent.spawn(subscriber1, router, resultSlot1)
gevent.spawn(subscriber2, router, resultSlot2)
gevent.sleep(0) # yield to subscribers for setup
gevent.spawn(publisher, router)
result1 = resultSlot1.wait()
result2 = resultSlot2.wait()
expected = [('foobar', 0),
('foobar', 1),
('foobar', 2)]
self.assertEqual(result1, expected)
self.assertEqual(result2, expected)
class LogListenerTask(Task):
def __init__(self, filter_, callback, contract_translator):
super(LogListenerTask, self).__init__()
self.filter_ = filter_
self.callback = callback
self.contract_translator = contract_translator
self.stop_event = AsyncResult()
self.sleep_time = 0.5
def _run(self): # pylint: disable=method-hidden
stop = None
while stop is None:
filter_changes = self.filter_.changes()
for log_event in filter_changes:
event = self.contract_translator.decode_event(
log_event['topics'],
log_event['data'],
)
if event is not None:
originating_contract = log_event['address']
self.callback(originating_contract, event)
stop = self.stop_event.wait(self.sleep_time)
def stop(self):
self.stop_event.set(True)
class ForwardSecretTask(Task):
timeout = TransferTask.timeout_per_hop
def __init__(self, transfermanager, hashlock, recipient):
self.transfermanager = transfermanager
self.recipient = recipient
self.hashlock = hashlock
self.raiden = transfermanager.assetmanager.raiden
super(ForwardSecretTask, self).__init__()
log.info("INIT", task=self)
self.transfermanager.on_task_started(self)
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address))
def _run(self):
self.event = AsyncResult() # http://www.gevent.org/gevent.event.html
timeout = self.timeout
msg = self.event.wait(timeout)
if not msg:
log.error("TIMEOUT! " * 5)
# TransferTimeout is of no use, SecretRequest was for sender
return self.on_completion(False)
assert isinstance(msg, Secret)
assert msg.hashlock == self.hashlock
fwd = Secret(msg.secret)
self.raiden.sign(fwd)
self.raiden.send(self.recipient, fwd)
return self.on_completion(True)
class LogListenerTask(Task):
def __init__(self, filter_, callback, contract_translator):
super(LogListenerTask, self).__init__()
self.filter_ = filter_
self.callback = callback
self.contract_translator = contract_translator
self.stop_event = AsyncResult()
self.sleep_time = 0.5
def _run(self): # pylint: disable=method-hidden
stop = None
while stop is None:
filter_changes = self.filter_.changes()
for log_event in filter_changes:
event = self.contract_translator.decode_event(
log_event['topics'],
log_event['data'],
)
if event is not None:
originating_contract = log_event['address']
self.callback(originating_contract, event)
stop = self.stop_event.wait(self.sleep_time)
def stop(self):
self.stop_event.set(True)
class ForwardSecretTask(gevent.Greenlet):
timeout = TransferTask.timeout_per_hop
def __init__(self, transfermanager, hashlock, recipient):
self.recipient = recipient
self.hashlock = hashlock
self.raiden = transfermanager.assetmanager.raiden
super(ForwardSecretTask, self).__init__()
print "INIT", self
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address))
def on_event(self, msg):
print "SET EVENT {} {} {}".format(self, id(self.event), msg)
if self.event.ready():
print "ALREADY HAD EVENT {} {} now {}".format(self, self.event.get(), msg)
assert self.event and not self.event.ready()
self.event.set(msg)
def _run(self):
self.event = AsyncResult() # http://www.gevent.org/gevent.event.html
timeout = self.timeout
msg = self.event.wait(timeout)
if not msg:
print "TIMEOUT! " * 5
# TransferTimeout is of no use, SecretRequest was for sender
return False
assert isinstance(msg, Secret)
assert msg.hashlock == self.hashlock
fwd = Secret(msg.secret)
self.raiden.sign(fwd)
self.raiden.send(self.recipient, fwd)
def asd():
dw = dolphinWatch.DolphinConnection()
connected_event = AsyncResult()
dw.onConnect(connected_event.set)
dw.connect()
connected_event.wait()
pointer = AsyncResult()
dw.read32(0x918F4FFC, pointer.set)
pointer = pointer.get()
print("Pointer: %x" % pointer)
#0x91951494
# phrase 0x5A9A4
# trainer name 0x5A97c
dw.writeMulti(pointer + 0x5A97c, [0x00, 0x4f, 0x00, 0x4e, 0x00, 0x45, 0x00, 0x48, 0x00, 0x41, 0x00, 0x4e, 0x00, 0x44, 0x00, 0x00])
def test_killing_unlinked(self):
e = AsyncResult()
def func():
try:
raise ExpectedError('test_killing_unlinked')
except:
e.set_exception(sys.exc_info()[1])
p = gevent.spawn_link(func)
try:
try:
e.wait()
except ExpectedError:
pass
finally:
p.unlink() # this disables LinkedCompleted that otherwise would be raised by the next line
sleep(DELAY)
class TestAsyncResult(object):
def __init__(self):
self.event = AsyncResult()
def run(self):
producers = [gevent.spawn(self._producer, i) for i in xrange(3)]
consumers = [gevent.spawn(self._consumer, i) for i in xrange(3)]
tasks = []
tasks.extend(producers)
tasks.extend(consumers)
gevent.joinall(tasks)
def _producer(self, pid):
print("I'm producer %d and now I don't want consume to do something" % (pid,))
sleeptime = random.randint(5, 10) * 0.01
print("Sleeping time is %f" % (sleeptime, ))
gevent.sleep(sleeptime)
print("I'm producer %d and now consumer could do something." % (pid,))
self.event.set('producer pid %d' % (pid, ))
def _consumer(self, pid):
print("I'm consumer %d and now I'm waiting for producer" % (pid,))
gevent.sleep(random.randint(0, 5) * 0.01)
value = self.event.wait()
print("I'm consumer %d. Value is %r and now I can do something" % (pid, value))
class ForwardSecretTask(Task):
def __init__(self, transfermanager, hashlock, recipient, msg_timeout):
self.transfermanager = transfermanager
self.recipient = recipient
self.hashlock = hashlock
self.msg_timeout = msg_timeout
self.raiden = transfermanager.raiden
super(ForwardSecretTask, self).__init__()
log.info('INIT', task=self)
self.transfermanager.on_task_started(self)
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address))
def _run(self): # pylint: disable=method-hidden
self.event = AsyncResult() # http://www.gevent.org/gevent.event.html
msg = self.event.wait(self.msg_timeout)
# returns None if msg_timeout is reached and event-value wasn't set()
if not msg:
log.error('TIMEOUT! ' * 5)
# TransferTimeout is of no use, SecretRequest was for sender
return self.on_completion(False)
assert isinstance(msg, Secret)
assert msg.hashlock == self.hashlock
fwd = Secret(msg.secret)
self.raiden.sign(fwd)
self.raiden.send(self.recipient, fwd)
return self.on_completion(True)
class ForwardSecretTask(Task):
timeout = TIMEOUT_PER_HOP
def __init__(self, transfermanager, hashlock, recipient):
self.transfermanager = transfermanager
self.recipient = recipient
self.hashlock = hashlock
self.raiden = transfermanager.raiden
super(ForwardSecretTask, self).__init__()
log.info('INIT', task=self)
self.transfermanager.on_task_started(self)
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__,
pex(self.raiden.address))
def _run(self): # pylint: disable=method-hidden
self.event = AsyncResult() # http://www.gevent.org/gevent.event.html
timeout = self.timeout
msg = self.event.wait(timeout)
if not msg:
log.error('TIMEOUT! ' * 5)
# TransferTimeout is of no use, SecretRequest was for sender
return self.on_completion(False)
assert isinstance(msg, Secret)
assert msg.hashlock == self.hashlock
fwd = Secret(msg.secret)
self.raiden.sign(fwd)
self.raiden.send(self.recipient, fwd)
return self.on_completion(True)
def wait(self, *args, **kwargs):
result = AsyncResult()
match = args[-1]
def _f(e):
if match(e):
result.set(e)
return result.wait(kwargs.pop('timeout', None))
def wait(self, *args, **kwargs):
result = AsyncResult()
match = args[-1]
def _f(e):
if match(e):
result.set(e)
return result.wait(kwargs.pop('timeout', None))
def test_killing_unlinked(self):
e = AsyncResult()
def func():
try:
raise ExpectedError('test_killing_unlinked')
except:
e.set_exception(sys.exc_info()[1])
p = gevent.spawn_link(func)
try:
try:
e.wait()
except ExpectedError:
pass
finally:
p.unlink(
) # this disables LinkedCompleted that otherwise would be raised by the next line
sleep(DELAY)
def once(self, *args, **kwargs):
result = AsyncResult()
li = None
def _f(e):
result.set(e)
li.detach()
li = self.on(*args + (_f, ))
return result.wait(kwargs.pop('timeout', None))
def _read_response(self, index, timeout):
rs = AsyncResult()
self._resps[index] = rs
resp = rs.wait(timeout)
self._resps.pop(index, None)
if not rs.successful():
error = rs.exception
if error is None:
error = Timeout
raise error
return resp
def once(self, *args, **kwargs):
result = AsyncResult()
li = None
def _f(e):
result.set(e)
li.detach()
li = self.on(*args + (_f, ))
return result.wait(kwargs.pop('timeout', None))
class EndMediatedTransferTask(Task):
""" Task that request a secret for a registered transfer. """
def __init__(self, transfermanager, originating_transfer):
super(EndMediatedTransferTask, self).__init__()
self.address = transfermanager.assetmanager.raiden.address
self.transfermanager = transfermanager
self.originating_transfer = originating_transfer
hashlock = originating_transfer.lock.hashlock
self.transfermanager.register_task_for_hashlock(self, hashlock)
def __repr__(self):
return '<{} {}>'.format(
self.__class__.__name__,
pex(self.address),
)
def _run(self): # pylint: disable=method-hidden
transfer = self.originating_transfer
assetmanager = self.transfermanager.assetmanager
raiden = assetmanager.raiden
transfer_details = '{} -> {} hash:{}'.format(
pex(transfer.target),
pex(transfer.initiator),
pex(transfer.hash),
)
log.debug('END MEDIATED TRANSFER {}'.format(transfer_details))
secret_request = SecretRequest(transfer.lock.hashlock)
raiden.sign(secret_request)
raiden.send(transfer.initiator, secret_request)
self.event = AsyncResult()
response = self.event.wait(raiden.config['msg_timeout'])
if response is None:
log.error('SECRETREQUEST TIMED OUT!')
self.transfermanager.on_hashlock_result(transfer.hashlock, False)
return
if not isinstance(response, Secret):
raise Exception('Invalid message received.')
if sha3(response.secret) != transfer.lock.hashlock:
raise Exception('Invalid secret received.')
# update all channels and propagate the secret
assetmanager.register_secret(response.secret)
self.transfermanager.on_hashlock_result(transfer.lock.hashlock, True)
class EndMediatedTransferTask(Task):
""" Task that request a secret for a registered transfer. """
def __init__(self, transfermanager, originating_transfer):
super(EndMediatedTransferTask, self).__init__()
self.address = transfermanager.assetmanager.raiden.address
self.transfermanager = transfermanager
self.originating_transfer = originating_transfer
hashlock = originating_transfer.lock.hashlock
self.transfermanager.register_task_for_hashlock(self, hashlock)
def __repr__(self):
return '<{} {}>'.format(
self.__class__.__name__,
pex(self.address),
)
def _run(self): # pylint: disable=method-hidden
transfer = self.originating_transfer
assetmanager = self.transfermanager.assetmanager
raiden = assetmanager.raiden
transfer_details = '{} -> {} hash:{}'.format(
pex(transfer.target),
pex(transfer.initiator),
pex(transfer.hash),
)
log.debug('END MEDIATED TRANSFER {}'.format(transfer_details))
secret_request = SecretRequest(transfer.lock.hashlock)
raiden.sign(secret_request)
raiden.send(transfer.initiator, secret_request)
self.event = AsyncResult()
response = self.event.wait(raiden.config['msg_timeout'])
if response is None:
log.error('SECRETREQUEST TIMED OUT!')
self.transfermanager.on_hashlock_result(transfer.hashlock, False)
return
if not isinstance(response, Secret):
raise Exception('Invalid message received.')
if sha3(response.secret) != transfer.lock.hashlock:
raise Exception('Invalid secret received.')
# update all channels and propagate the secret
assetmanager.register_secret(response.secret)
self.transfermanager.on_hashlock_result(transfer.lock.hashlock, True)
def call(self, fname, data, noresult=False, timeout=None):
""" server -> client: rpc call """
gw = self.gw_mgr.rpc_gws[self.gwid]
data = ujson.dumps({FIELD_FUNC: fname,
FIELD_STATUS: 1,
FIELD_DATA: data,
FIELD_ERROR: ''})
mid = next(self._call_id)
gw.send(self.pid, self.route_id, b'%s%s' % (struct.pack(DATA_HEAD_STRUCT, TAG_REQ, mid), bytes(data, 'utf-8')))
if noresult:
return
result = AsyncResult()
self._waits[mid] = result
return result.wait(timeout)
class LogListenerTask(Task):
def __init__(self, listener_name, filter_, callback, contract_translator,
events_poll_timeout=DEFAULT_EVENTS_POLL_TIMEOUT):
super(LogListenerTask, self).__init__()
self.listener_name = listener_name
self.filter_ = filter_
self.callback = callback
self.contract_translator = contract_translator
self.stop_event = AsyncResult()
self.sleep_time = events_poll_timeout
# exposes the AsyncResult timer, this allows us to raise the timeout
# inside this Task to force an update:
#
# task.kill(task.timeout)
#
self.timeout = None
def __repr__(self):
return '<LogListenerTask {}>'.format(self.listener_name)
def _run(self): # pylint: disable=method-hidden
stop = None
while stop is None:
filter_changes = self.filter_.changes()
for log_event in filter_changes:
log.debug('New Events', task=self.listener_name)
event = self.contract_translator.decode_event(
log_event['topics'],
log_event['data'],
)
if event is not None:
originating_contract = log_event['address']
try:
self.callback(originating_contract, event)
except:
log.exception('unexpected exception on log listener')
self.timeout = Timeout(self.sleep_time) # wait() will call cancel()
stop = self.stop_event.wait(self.timeout)
def stop(self):
self.stop_event.set(True)
def test_heartbeat_with_listeners(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name,
listeners=[mocklistener],
service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
mocklistener.listen = lambda *a, **kw: fake_listen(
listenoutev, listeninev)
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(
listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
# now test heartbeat!
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def test_heartbeat_listener_dead(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name,
listeners=[mocklistener],
service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(
listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
listeninev.set() # stop the listen loop
p.thread_manager.children[1].join(
timeout=5) # wait for listen loop to terminate
hb = p.heartbeat()
self.assertEquals((False, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
class RunnableTest(Runnable):
def __init__(self):
super().__init__()
def start(self):
self._stop_event = AsyncResult()
super().start()
def _run(self, *args: Any, **kwargs: Any) -> None:
while self._stop_event and self._stop_event.wait(0.5) is not True:
gevent.sleep(0.1)
return
def stop(self):
if self._stop_event:
self._stop_event.set(True)
class BaseConnection(object):
"""A connection to an AMQP server.
This class deals with establishing and reconnecting errors, and routes
messages received off the wire to handlers registered with the
corresponding channel. It also handles sending and receiving heartbeat
frames to detect when the connection has been lost.
AMQP sends connection-level errors (connection.close) that cause the
connection to close; to support this we dispatch such errors to all
channels.
"""
frame_max = 131072 # adjusted by Tune frame
channel_max = 65535 # adjusted by Tune method
MAX_SEND_QUEUE = 32 # frames
server_properties = None # properties and capabilities sent by the remote connection
heartbeat = 0 # seconds between heartbeats. 0 means off.
# NB. heartbeats don't start until they are negotiated
def __init__(self, amqp_url='amqp:///', heartbeat=30, debug=False):
super(BaseConnection, self).__init__()
self.channel_id = 0
self.channels = {}
self.connect_lock = RLock()
self.channels_lock = RLock()
self.queue = None
self.state = STATE_DISCONNECTED
self.disconnect_event = Event()
self.debug = debug
# Negotiate for heartbeats
self.requested_heartbeat = heartbeat
(self.username, self.password, self.vhost, self.host, self.port) = \
parse_amqp_url(str(amqp_url))
def allocate_channel(self):
"""Create a new channel."""
self.connect() # This is a no-op if we're already connected
with self.channels_lock:
for i in xrange(self.channel_max):
self.channel_id = self.channel_id % (self.channel_max - 1) + 1
if self.channel_id not in self.channels:
break
else:
raise ChannelError("No available channels!")
id = self.channel_id
chan = MessageChannel(self, id)
self.channels[id] = chan
chan.channel_open()
return chan
def _remove_channel(self, id):
"""Remove a channel (presumably because it has closed.)"""
with self.channels_lock:
del(self.channels[id])
def connected(self):
return self.state == STATE_CONNECTED
def connect(self):
"""Open the connection to the server.
This method blocks until the connection has been opened and handshaking
is complete.
If connection fails, an exception will be raised instead.
"""
# Don't bother acquiring the lock if we are already connected
if self.connected():
return
with self.connect_lock:
# Check again now we've acquired the lock, as the previous holder
# of the lock would typically have completed the connection.
if self.connected():
return
self.disconnect_event.clear()
self.connected_event = AsyncResult()
self._connect()
v = self.connected_event.wait() # Block until the connection is
# properly ready
# FFS, AsyncResult.set_exception doesn't work in gevent 1.0b4
# so we just use normal setting, but with exception types
if isinstance(v, Exception):
raise v
def _connect(self):
"""Connect to the remote server and start reader/writer greenlets."""
# NB. this method isn't safe to call directly; must only be called
# via connect() which adds the necessary locking
self.state = STATE_CONNECTING
try:
try:
addrinfo = socket.getaddrinfo(self.host, self.port, socket.AF_INET6, socket.SOCK_STREAM)
except socket.gaierror:
addrinfo = socket.getaddrinfo(self.host, self.port, socket.AF_INET, socket.SOCK_STREAM)
(family, socktype, proto, canonname, sockaddr) = addrinfo[0]
self.sock = socket.socket(family, socktype, proto)
self.sock.connect(sockaddr)
except:
self.state = STATE_DISCONNECTED
raise
# Set up channel state
self.channel_id = 1
self.channels = {}
# Set up connection greenlets
self.queue = Queue(self.MAX_SEND_QUEUE)
reader = gevent.spawn(self.do_read, self.sock)
writer = gevent.spawn(self.do_write, self.sock, self.queue)
reader.link(lambda reader: writer.kill())
writer.link(lambda writer: reader.kill())
self.reader = reader
self.writer = writer
def _on_error(self, exc):
"""Dispatch a connection error to all channels."""
for id, channel in self.channels.items():
channel.on_error(exc)
self.fire_async('error')
def _on_connect(self):
"""Called when the connection is fully open."""
self.state = STATE_CONNECTED
self.connected_event.set("Connected!")
self.fire_async('connect')
def _on_abnormal_disconnect(self, exc):
"""Called when the connection has been abnormally disconnected."""
self.state = STATE_DISCONNECTED
try:
self._on_error(exc)
except Exception:
pass
while True:
try:
self.connect()
except Exception:
gevent.sleep(5)
else:
break
def _on_normal_disconnect(self):
"""Called when the connection has closed."""
self.state = STATE_DISCONNECTED
self._on_error(ConnectionError("Connection closed."))
self.queue.put(None)
self.disconnect_event.set()
def do_read(self, sock):
"""Run a reader greenlet.
This method will read a preamble then loop forever reading frames off
the wire and dispatch them to channels.
"""
try:
reader = BufferedReader(sock)
TIMEOUT_EXC = ConnectionError('Heartbeat timeout')
while True:
if self.heartbeat:
t = gevent.Timeout(
seconds=self.heartbeat * 2,
exception=TIMEOUT_EXC)
t.start()
else:
t = None
frame_header = reader.read(FRAME_HEADER.size)
frame_type, channel, size = \
FRAME_HEADER.unpack(frame_header)
payload = reader.read(size + 1)
if t is not None:
t.cancel()
if payload[-1] != '\xCE':
raise ConnectionError('Received invalid frame data')
if self.debug:
self._debug_print('s->c', frame_header + payload)
buffer = DecodeBuffer(payload)
if frame_type == 0x01: # Method frame
method_id, = buffer.read('!I')
frame = spec.METHODS[method_id].decode(buffer)
self.inbound_method(channel, frame)
elif frame_type == 0x02: # header frame
class_id, body_size = buffer.read('!HxxQ')
props = spec.PROPS[class_id](buffer)
self.inbound_props(channel, body_size, props)
elif frame_type == 0x03: # body frame
self.inbound_body(channel, payload[:-1])
elif frame_type in [0x04, 0x08]:
# Heartbeat frame
#
# Catch it as both 0x04 and 0x08 - see
# http://www.rabbitmq.com/amqp-0-9-1-errata.html#section_29
#
# We don't need to handle this specifically - it's already
# covered by the read timeout above.
pass
else:
raise ConnectionError("Unknown frame type")
except (gevent.GreenletExit, Exception) as e:
self.connected_event.set(e)
if self.state in [STATE_CONNECTED, STATE_CONNECTING]:
# Spawn a new greenlet to run the reconnect loop
gevent.spawn(self._on_abnormal_disconnect, e)
else:
self.state = STATE_DISCONNECTED
def inbound_method(self, channel, frame):
"""Dispatch an inbound method."""
try:
c = self.channels[channel]
except KeyError:
if frame.name == 'connection.start':
c = StartChannel(self, channel)
self.channels[channel] = c
else:
return
c._on_method(frame)
def inbound_props(self, channel, body_size, props):
"""Dispatch an inbound properties frame."""
try:
c = self.channels[channel]
except KeyError:
return
c._on_headers(body_size, props)
def inbound_body(self, channel, payload):
"""Dispatch an inbound body frame."""
try:
c = self.channels[channel]
except KeyError:
return
c._on_body(payload)
def do_write(self, sock, queue):
"""Run a writer greenlet.
This greenlet will loop until the connection closes, writing frames
from the queue.
"""
# Write the protocol header
sock.sendall(spec.PREAMBLE)
# Enter a send loop
while True:
if self.heartbeat:
try:
msg = queue.get(timeout=self.heartbeat)
except Empty:
# Send heartbeat
sock.sendall(
FRAME_HEADER.pack(0x08, 0, 0) + '\xCE'
)
continue
else:
msg = queue.get()
if msg is None:
return
if self.debug:
self._debug_print('s<-c', msg)
sock.sendall(msg)
def _debug_print(self, direction, msg):
"""Decode a frame and print it for debugging."""
try:
# Print method, for debugging
type, channel, size = FRAME_HEADER.unpack_from(msg)
if type == 1:
method_id = struct.unpack_from('!I', msg, FRAME_HEADER.size)[0]
print direction, spec.METHODS[method_id].name
else:
print direction, {
2: '[headers %d bytes]',
3: '[payload %d bytes]',
4: '[heartbeat %d bytes]',
}[type] % size
except Exception:
import traceback
traceback.print_exc()
def _send_frames(self, channel, frames):
"""Send a sequence of frames on channel.
Each frame will be put onto a queue for the writer to write, which
could cause the calling greenlet to block if the queue is full.
This should cause large outgoing messages to be spliced together so
that no caller is starved of service while a large message is sending.
"""
assert channel in self.channels
for type, payload in frames:
fdata = ''.join([
FRAME_HEADER.pack(type, channel, len(payload)),
payload,
'\xCE'
])
self.queue.put(fdata)
def _tune(self, frame_max, channel_max, heartbeat=0):
"""Adjust connection parameters.
Called in response to negotiation with the server.
"""
frame_max = frame_max if frame_max != 0 else 2**19
# limit the maximum frame size, to ensure messages are multiplexed
self.frame_max = min(131072, frame_max)
self.channel_max = channel_max if channel_max > 0 else 65535
if heartbeat:
self.heartbeat = min(self.requested_heartbeat, heartbeat)
else:
self.heartbeat = self.requested_heartbeat
def close(self, block=True, timeout=2):
if self.state in [STATE_CONNECTED, STATE_CONNECTING]:
self.state = STATE_DISCONNECTING
if block:
self.channels[0].connection_close()
self.writer.join(timeout=timeout)
else:
self.channels[0]._send(
spec.FrameConnectionClose(200, '', 0, 0)
)
def set_dispatcher(self, dispatcher):
"""Set the event dispatcher.
BaseConnection holds a weak ref to its dispatcher so that our
connection threads don't keep it alive.
"""
self._dispatcher = weakref.ref(dispatcher)
def get_dispatcher(self):
"""Get the event dispatcher or return None if there is no dispatcher."""
if self._dispatcher:
return self._dispatcher()
dispatcher = property(get_dispatcher, set_dispatcher)
def fire(self, event, *args, **kwargs):
"""Fire an event using the dispatcher."""
d = self.dispatcher
if d:
return self.dispatcher.fire(event, *args, **kwargs)
def fire_async(self, event, *args, **kwargs):
"""Asynchronously fire an event using the dispatcher."""
d = self.dispatcher
if d:
return self.dispatcher.fire_async(event, *args, **kwargs)
class MediatedTransferTask(Task):
# Normal Operation (Transfer A > C)
# A: Initiator Creates Secret
# A: MediatedTransfer > B
# B: MediatedTransfer > C
# C: SecretRequest > A (implicitly signs, that valid transfer was received)
# A: Secret > C
# C: Secret > B
# Timeout (Transfer A > C)
# A: Initiator Creates Secret
# A: MediatedTransfer > B
# B: MediatedTransfer > C
# Failure: No Ack from C
# B: TransferTimeout > A
# Resolution: A won't reveal the secret, tries new transfer, B bans C
# CancelTransfer (Transfer A > D)
# A: Initiator Creates Secret
# A: MediatedTransfer > B
# B: MediatedTransfer > C
# Failure: C can not establish path to D (e.g. insufficient distributable, no active node)
# C: CancelTransfer > B (levels out balance)
# B: MediatedTransfer > C2
# C2: MediatedTransfer > D
def __init__(self, transfermanager, amount, target, hashlock, expiration,
originating_transfer=None, secret=None): # fee!
import transfermanager as transfermanagermodule
assert isinstance(transfermanager, transfermanagermodule.TransferManager)
self.amount = amount
self.assetmanager = transfermanager.assetmanager
self.event = None
self.fee = 0 # FIXME: calculate fee
self.hashlock = hashlock
self.expiration = expiration
self.originating_transfer = originating_transfer
self.raiden = transfermanager.raiden
self.secret = secret
self.target = target
self.transfermanager = transfermanager
self.isinitiator = bool(secret)
if originating_transfer and secret:
raise ValueError('Cannot set both secret and originating_transfer')
if not (originating_transfer or secret):
raise ValueError('Either originating_transfer or secret needs to be informed')
if originating_transfer and not isinstance(originating_transfer, LockedTransfer):
raise ValueError('originating_transfer needs to be a LockedTransfer')
if self.isinitiator:
self.initiator = self.raiden.address
else:
self.initiator = originating_transfer.initiator
super(MediatedTransferTask, self).__init__()
self.transfermanager.on_task_started(self)
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address))
def _run(self): # pylint: disable=method-hidden
for path, channel in self.get_best_routes():
next_hop = path[1]
mediated_transfer = channel.create_mediatedtransfer(
self.initiator,
self.target,
self.fee,
self.amount,
# HOTFIX: you cannot know channel.settle_timeout beforehand,since path is not yet defined
# FIXME: implement expiration adjustment in different task (e.g. 'InitMediatedTransferTask')
self.expiration if self.expiration is not None else channel.settle_timeout - 1,
self.hashlock,
)
self.raiden.sign(mediated_transfer)
channel.register_transfer(mediated_transfer)
log.debug('MEDIATED TRANSFER initiator={} {}'.format(
pex(mediated_transfer.initiator),
lpex(path),
))
msg_timeout = self.raiden.config['msg_timeout']
# timeout not dependent on expiration (canceltransfer/transfertimeout msgs),
# but should be set shorter than the expiration
msg = self.send_transfer_and_wait(next_hop, mediated_transfer, path, msg_timeout)
log.debug('MEDIATED TRANSFER RETURNED {} {}'.format(
pex(self.raiden.address),
msg,
))
result = self.check_path(msg, channel)
# `next_hop` didn't get a response from any of it's, let's try the
# next path
if isinstance(result, TransferTimeout):
continue
# `next_hop` doesn't have any path with a valid channel to proceed,
# try the next path
if isinstance(result, CancelTransfer):
continue
# `check_path` failed, try next path
if result is None:
continue
return self.on_completion(result)
# No suitable path avaiable (e.g. insufficient distributable, no active node)
# Send CancelTransfer to the originating node, this has the effect of
# backtracking in the graph search of the raiden network.
if self.originating_transfer:
from_address = self.originating_transfer.sender
from_transfer = self.originating_transfer
from_channel = self.assetmanager.channels[from_address]
log.debug('CANCEL MEDIATED TRANSFER from={} {}'.format(
pex(from_address),
pex(self.raiden.address),
))
cancel_transfer = from_channel.create_canceltransfer_for(from_transfer)
self.raiden.sign(cancel_transfer)
from_channel.register_transfer(cancel_transfer)
self.raiden.send(from_address, cancel_transfer)
else:
log.error('UNABLE TO COMPLETE MEDIATED TRANSFER target={} amount={}'.format(
pex(self.target),
self.amount,
))
return self.on_completion(False)
def get_best_routes(self):
""" Yield a two-tuple (path, channel) that can be used to mediate the
transfer. The result is ordered from the best to worst path.
"""
available_paths = self.assetmanager.channelgraph.get_shortest_paths(
self.raiden.address,
self.target,
)
for path in available_paths:
assert path[0] == self.raiden.address
assert path[1] in self.assetmanager.channels
assert path[-1] == self.target
partner = path[1]
channel = self.assetmanager.channels[partner]
if not channel.isopen:
continue
# we can't intermediate the transfer if we don't have enough funds
if self.amount > channel.distributable:
continue
# Our partner won't accept a locked transfer that can expire after
# the settlement period, otherwise the secret could be revealed
# after channel is settled and he would lose the asset, or before
# the minimum required.
# FIXME Hotfix, see self._run()
if not self.expiration:
expiration = channel.settle_timeout - 1
else:
expiration = self.expiration
if not (self.raiden.chain.block_number + channel.reveal_timeout <=
expiration <
channel.settle_timeout):
log.debug(
'expiration is too large, channel/path cannot be used',
expiration=expiration,
channel_locktime=channel.settle_timeout,
nodeid=pex(path[0]),
partner=pex(path[1]),
)
continue
yield (path, channel)
def check_path(self, msg, channel):
if isinstance(msg, CancelTransfer):
return None # try with next path
elif isinstance(msg, TransferTimeout):
# stale hashlock
if not self.isinitiator:
self.raiden.send(self.originating_transfer.sender, msg)
return False
elif isinstance(msg, Secret):
assert self.originating_transfer
assert msg.hashlock == self.hashlock
if self.originating_transfer.sender != self.originating_transfer.initiator:
fwd = Secret(msg.secret)
self.raiden.sign(fwd)
self.raiden.send(self.originating_transfer.sender, fwd)
else:
log.warning('NOT FORWARDING SECRET TO ININTIATOR')
return True
elif isinstance(msg, SecretRequest):
assert self.isinitiator
# lock.target can easilly be tampered, ensure that we are receiving
# the SecretRequest from the correct node
if msg.sender != self.target:
log.error('Tampered SecretRequest', secret_request=msg)
return None # try the next available path
# TODO: the lock.amount can easily be tampered, check the `target`
# locked transfer has the correct `amount`
msg = Secret(self.secret)
self.raiden.sign(msg)
self.raiden.send(self.target, msg)
# TODO: Guarantee that `target` received the secret, otherwise we
# updated the channel and the first hop will receive the asset, but
# none of the other channels will make the transfer
channel.claim_locked(self.secret)
return True
return None
def send_transfer_and_wait(self, recipient, transfer, path, msg_timeout):
""" Send `transfer` to `recipient` and wait for the response.
Args:
recipient (address): The address of the node that will receive the
message.
transfer: The transfer message.
path: The current path that is being tried.
msg_timeout: How long should we wait for a response from `recipient`.
Returns:
TransferTimeout: If the other end didn't respond
"""
self.event = AsyncResult()
self.raiden.send(recipient, transfer)
# The event is set either when a relevant message is received or we
# reach the timeout.
#
# The relevant messages are: CancelTransfer, TransferTimeout,
# SecretRequest, or Secret
msg = self.event.wait(msg_timeout)
# Timed out
if msg is None:
log.error('TIMEOUT [{}]! recipient={} didnt respond path={}'.format(
msg_timeout,
pex(recipient),
lpex(path),
))
transfer_timeout = TransferTimeout(
echo=transfer.hash,
hashlock=transfer.lock.hashlock,
)
self.raiden.sign(transfer_timeout)
return transfer_timeout
log.debug(
'HAVE EVENT {} {}'.format(self, msg),
node=pex(self.raiden.address),
)
if isinstance(msg, CancelTransfer):
assert msg.lock.hashlock == transfer.lock.hashlock
assert msg.lock.amount == transfer.lock.amount
assert msg.recipient == transfer.sender == self.raiden.address
channel = self.assetmanager.channels[msg.sender]
channel.register_transfer(msg)
return msg
elif isinstance(msg, TransferTimeout):
assert msg.echo == transfer.hash
return msg
# send back StaleHashLock, we need new hashlock
elif isinstance(msg, Secret):
# done exit
assert msg.hashlock == self.hashlock
# channel = self.assetmanager.channels[msg.recipient]
# channel.claim_locked(msg.secret) # fixme this is also done by assetmanager
return msg
elif isinstance(msg, SecretRequest):
# reveal secret
log.info('SECRETREQUEST RECEIVED {}'.format(msg))
assert msg.sender == self.target
return msg
raise NotImplementedError()
class Popen(SubprocessProtocol):
def __init__(self, args, bufsize=-1,
stdin=None, stdout=None, stderr=None,
shell=False, universal_newlines=False, **kwargs):
"""Create new Popen instance."""
assert not universal_newlines, "universal_newlines must be False"
hub = get_hub()
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
self.stdin = None
self.stdout = None
self.stderr = None
self._transport = None
if shell:
hub.wait_async(hub.loop.subprocess_shell(
lambda: _DelegateProtocol(self), *args,
stdin=stdin, stdout=stdout, stderr=stderr,
bufsize=bufsize, **kwargs))
else:
hub.wait_async(hub.loop.subprocess_exec(
lambda: _DelegateProtocol(self), *args,
stdin=stdin, stdout=stdout, stderr=stderr,
bufsize=bufsize, **kwargs))
def __repr__(self):
return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode)
def communicate(self, input=None, timeout=None):
"""Interact with process: Send data to stdin. Read data from
stdout and stderr, until end-of-file is reached. Wait for
process to terminate. The optional input argument should be a
string to be sent to the child process, or None, if no data
should be sent to the child.
communicate() returns a tuple (stdout, stderr)."""
#TODO: timeout
greenlets = []
if self.stdin:
greenlets.append(spawn(write_and_close, self.stdin, input))
if self.stdout:
stdout = spawn(self.stdout.read)
greenlets.append(stdout)
else:
stdout = None
if self.stderr:
stderr = spawn(self.stderr.read)
greenlets.append(stderr)
else:
stderr = None
joinall(greenlets)
if self.stdout:
self.stdout.close()
if self.stderr:
self.stderr.close()
self.wait()
return (None if stdout is None else stdout.value or '',
None if stderr is None else stderr.value or '')
def poll(self):
return self._internal_poll()
def rawlink(self, callback):
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def send_signal(self, sig):
self._transport.send_signal(sig)
def terminate(self):
self._transport.terminate()
def kill(self):
#noinspection PyProtectedMember
if self._transport._proc is not None:
self._transport.kill()
if mswindows:
#
# Windows methods
#
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
self.returncode = GetExitCodeProcess(self._handle)
self.result.set(self.returncode)
return self.returncode
def rawlink(self, callback):
if not self.result.ready() and not self._waiting:
self._waiting = True
Greenlet.spawn(self._wait)
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _blocking_wait(self):
WaitForSingleObject(self._handle, INFINITE)
self.returncode = GetExitCodeProcess(self._handle)
return self.returncode
def _wait(self):
self.threadpool.spawn(self._blocking_wait).rawlink(self.result)
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
if self.returncode is None:
if not self._waiting:
self._waiting = True
self._wait()
return self.result.wait(timeout=timeout)
else:
#
# POSIX methods
#
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if get_hub() is not getcurrent():
sig_pending = getattr(self._loop, 'sig_pending', True)
if sig_pending:
sleep(0.00001)
return self.returncode
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
return self.result.wait(timeout=timeout)
class Popen(object):
def __init__(self, args, bufsize=None, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None,
**kwargs):
"""Create new Popen instance."""
if not PY3 and kwargs:
raise TypeError("Got unexpected keyword arguments", kwargs)
pass_fds = kwargs.pop('pass_fds', ())
start_new_session = kwargs.pop('start_new_session', False)
restore_signals = kwargs.pop('restore_signals', True)
hub = get_hub()
if bufsize is None:
# bufsize has different defaults on Py3 and Py2
if PY3:
bufsize = -1
else:
bufsize = 0
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
any_stdio_set = (stdin is not None or stdout is not None or
stderr is not None)
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
if any_stdio_set:
close_fds = False
else:
close_fds = True
elif close_fds and any_stdio_set:
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
# close_fds has different defaults on Py3/Py2
if PY3:
close_fds = True
else:
close_fds = False
if pass_fds and not close_fds:
import warnings
warnings.warn("pass_fds overriding close_fds.", RuntimeWarning)
close_fds = True
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
if PY3:
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
# We wrap OS handles *before* launching the child, otherwise a
# quickly terminating child could make our fds unwrappable
# (see #8458).
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
if PY3 and universal_newlines:
# Under Python 3, if we left on the 'b' we'd get different results
# depending on whether we used FileObjectPosix or FileObjectThread
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
self.stdin._translate = True
self.stdin.io = io.TextIOWrapper(self.stdin.io, write_through=True,
line_buffering=(bufsize == 1))
else:
self.stdin = FileObject(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
if PY3:
# FileObjectThread doesn't support the 'U' qualifier
# with a bufsize of 0
self.stdout = FileObject(c2pread, 'rb', bufsize)
# NOTE: Universal Newlines are broken on Windows/Py3, at least
# in some cases. This is true in the stdlib subprocess module
# as well; the following line would fix the test cases in
# test__subprocess.py that depend on python_universal_newlines,
# but would be inconsistent with the stdlib:
#msvcrt.setmode(self.stdout.fileno(), os.O_TEXT)
self.stdout.io = io.TextIOWrapper(self.stdout.io)
self.stdout.io.mode = 'r'
self.stdout._translate = True
else:
self.stdout = FileObject(c2pread, 'rU', bufsize)
else:
self.stdout = FileObject(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
if PY3:
self.stderr = FileObject(errread, 'rb', bufsize)
self.stderr.io = io.TextIOWrapper(self.stderr.io)
self.stderr._translate = True
else:
self.stderr = FileObject(errread, 'rU', bufsize)
else:
self.stderr = FileObject(errread, 'rb', bufsize)
self._closed_child_pipe_fds = False
try:
self._execute_child(args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session)
except:
# Cleanup if the child failed starting.
# (gevent: New in python3, but reported as gevent bug in #347.
# Note that under Py2, any error raised below will replace the
# original error so we have to use reraise)
if not PY3:
exc_info = sys.exc_info()
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
except (OSError, IOError):
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
to_close = []
if stdin == PIPE:
to_close.append(p2cread)
if stdout == PIPE:
to_close.append(c2pwrite)
if stderr == PIPE:
to_close.append(errwrite)
if hasattr(self, '_devnull'):
to_close.append(self._devnull)
for fd in to_close:
try:
os.close(fd)
except (OSError, IOError):
pass
if not PY3:
try:
reraise(*exc_info)
finally:
del exc_info
raise
def __repr__(self):
return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode)
def _on_child(self, watcher):
watcher.stop()
status = watcher.rstatus
if os.WIFSIGNALED(status):
self.returncode = -os.WTERMSIG(status)
else:
self.returncode = os.WEXITSTATUS(status)
self.result.set(self.returncode)
def _get_devnull(self):
if not hasattr(self, '_devnull'):
self._devnull = os.open(os.devnull, os.O_RDWR)
return self._devnull
_stdout_buffer = None
_stderr_buffer = None
def communicate(self, input=None, timeout=None):
"""Interact with process: Send data to stdin. Read data from
stdout and stderr, until end-of-file is reached. Wait for
process to terminate. The optional input argument should be a
string to be sent to the child process, or None, if no data
should be sent to the child.
communicate() returns a tuple (stdout, stderr).
:keyword timeout: Under Python 2, this is a gevent extension; if
given and it expires, we will raise :class:`gevent.timeout.Timeout`.
Under Python 3, this raises the standard :exc:`TimeoutExpired` exception.
"""
greenlets = []
if self.stdin:
greenlets.append(spawn(write_and_close, self.stdin, input))
# If the timeout parameter is used, and the caller calls back after
# getting a TimeoutExpired exception, we can wind up with multiple
# greenlets trying to run and read from and close stdout/stderr.
# That's bad because it can lead to 'RuntimeError: reentrant call in io.BufferedReader'.
# We can't just kill the previous greenlets when a timeout happens,
# though, because we risk losing the output collected by that greenlet
# (and Python 3, where timeout is an official parameter, explicitly says
# that no output should be lost in the event of a timeout.) Instead, we're
# watching for the exception and ignoring it. It's not elegant,
# but it works
if self.stdout:
def _read_out():
try:
data = self.stdout.read()
except RuntimeError:
return
if self._stdout_buffer is not None:
self._stdout_buffer += data
else:
self._stdout_buffer = data
stdout = spawn(_read_out)
greenlets.append(stdout)
else:
stdout = None
if self.stderr:
def _read_err():
try:
data = self.stderr.read()
except RuntimeError:
return
if self._stderr_buffer is not None:
self._stderr_buffer += data
else:
self._stderr_buffer = data
stderr = spawn(_read_err)
greenlets.append(stderr)
else:
stderr = None
# If we were given stdin=stdout=stderr=None, we have no way to
# communicate with the child, and thus no greenlets to wait
# on. This is a nonsense case, but it comes up in the test
# case for Python 3.5 (test_subprocess.py
# RunFuncTestCase.test_timeout). Instead, we go directly to
# self.wait
if not greenlets and timeout is not None:
result = self.wait(timeout=timeout)
# Python 3 would have already raised, but Python 2 would not
# so we need to do that manually
if result is None:
from gevent.timeout import Timeout
raise Timeout(timeout)
done = joinall(greenlets, timeout=timeout)
if timeout is not None and len(done) != len(greenlets):
if PY3:
raise TimeoutExpired(self.args, timeout)
from gevent.timeout import Timeout
raise Timeout(timeout)
if self.stdout:
try:
self.stdout.close()
except RuntimeError:
pass
if self.stderr:
try:
self.stderr.close()
except RuntimeError:
pass
self.wait()
stdout_value = self._stdout_buffer
self._stdout_buffer = None
stderr_value = self._stderr_buffer
self._stderr_buffer = None
# XXX: Under python 3 in universal newlines mode we should be
# returning str, not bytes
return (None if stdout is None else stdout_value or b'',
None if stderr is None else stderr_value or b'')
def poll(self):
return self._internal_poll()
if PY3:
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if self.stdout:
self.stdout.close()
if self.stderr:
self.stderr.close()
try: # Flushing a BufferedWriter may raise an error
if self.stdin:
self.stdin.close()
finally:
# Wait for the process to terminate, to avoid zombies.
# JAM: gevent: If the process never terminates, this
# blocks forever.
self.wait()
if mswindows:
#
# Windows methods
#
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin is None and stdout is None and stderr is None:
return (None, None, None, None, None, None)
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
try:
DEVNULL
except NameError:
_devnull = object()
else:
_devnull = DEVNULL
if stdin is None:
p2cread = GetStdHandle(STD_INPUT_HANDLE)
if p2cread is None:
p2cread, _ = CreatePipe(None, 0)
if PY3:
p2cread = Handle(p2cread)
_winapi.CloseHandle(_)
elif stdin == PIPE:
p2cread, p2cwrite = CreatePipe(None, 0)
if PY3:
p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
elif stdin == _devnull:
p2cread = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stdin, int):
p2cread = msvcrt.get_osfhandle(stdin)
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout is None:
c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
if c2pwrite is None:
_, c2pwrite = CreatePipe(None, 0)
if PY3:
c2pwrite = Handle(c2pwrite)
_winapi.CloseHandle(_)
elif stdout == PIPE:
c2pread, c2pwrite = CreatePipe(None, 0)
if PY3:
c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
elif stdout == _devnull:
c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stdout, int):
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr is None:
errwrite = GetStdHandle(STD_ERROR_HANDLE)
if errwrite is None:
_, errwrite = CreatePipe(None, 0)
if PY3:
errwrite = Handle(errwrite)
_winapi.CloseHandle(_)
elif stderr == PIPE:
errread, errwrite = CreatePipe(None, 0)
if PY3:
errread, errwrite = Handle(errread), Handle(errwrite)
elif stderr == STDOUT:
errwrite = c2pwrite
elif stderr == _devnull:
errwrite = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stderr, int):
errwrite = msvcrt.get_osfhandle(stderr)
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _make_inheritable(self, handle):
"""Return a duplicate of handle, which is inheritable"""
return DuplicateHandle(GetCurrentProcess(),
handle, GetCurrentProcess(), 0, 1,
DUPLICATE_SAME_ACCESS)
def _find_w9xpopen(self):
"""Find and return absolut path to w9xpopen.exe"""
w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
# Eeek - file-not-found - possibly an embedding
# situation - see if we can locate it in sys.exec_prefix
w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
raise RuntimeError("Cannot locate w9xpopen.exe, which is "
"needed for Popen to work with your "
"shell or platform.")
return w9xpopen
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
unused_restore_signals, unused_start_new_session):
"""Execute program (MS Windows version)"""
assert not pass_fds, "pass_fds not supported on Windows."
if not isinstance(args, string_types):
args = list2cmdline(args)
# Process startup details
if startupinfo is None:
startupinfo = STARTUPINFO()
if None not in (p2cread, c2pwrite, errwrite):
startupinfo.dwFlags |= STARTF_USESTDHANDLES
startupinfo.hStdInput = p2cread
startupinfo.hStdOutput = c2pwrite
startupinfo.hStdError = errwrite
if shell:
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
startupinfo.wShowWindow = SW_HIDE
comspec = os.environ.get("COMSPEC", "cmd.exe")
args = '{} /c "{}"'.format(comspec, args)
if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com":
# Win9x, or using command.com on NT. We need to
# use the w9xpopen intermediate program. For more
# information, see KB Q150956
# (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
w9xpopen = self._find_w9xpopen()
args = '"%s" %s' % (w9xpopen, args)
# Not passing CREATE_NEW_CONSOLE has been known to
# cause random failures on win9x. Specifically a
# dialog: "Your program accessed mem currently in
# use at xxx" and a hopeful warning about the
# stability of your system. Cost is Ctrl+C wont
# kill children.
creationflags |= CREATE_NEW_CONSOLE
# Start the process
try:
hp, ht, pid, tid = CreateProcess(executable, args,
# no special security
None, None,
int(not close_fds),
creationflags,
env,
cwd,
startupinfo)
except IOError as e: # From 2.6 on, pywintypes.error was defined as IOError
# Translate pywintypes.error to WindowsError, which is
# a subclass of OSError. FIXME: We should really
# translate errno using _sys_errlist (or similar), but
# how can this be done from Python?
if PY3:
raise # don't remap here
raise WindowsError(*e.args)
finally:
# Child is launched. Close the parent's copy of those pipe
# handles that only the child should have open. You need
# to make sure that no handles to the write end of the
# output pipe are maintained in this process or else the
# pipe will not close when the child process exits and the
# ReadFile will hang.
def _close(x):
if x is not None and x != -1:
if hasattr(x, 'Close'):
x.Close()
else:
_winapi.CloseHandle(x)
_close(p2cread)
_close(c2pwrite)
_close(errwrite)
if hasattr(self, '_devnull'):
os.close(self._devnull)
# Retain the process handle, but close the thread handle
self._child_created = True
self._handle = Handle(hp) if not hasattr(hp, 'Close') else hp
self.pid = pid
_winapi.CloseHandle(ht) if not hasattr(ht, 'Close') else ht.Close()
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
self.returncode = GetExitCodeProcess(self._handle)
self.result.set(self.returncode)
return self.returncode
def rawlink(self, callback):
if not self.result.ready() and not self._waiting:
self._waiting = True
Greenlet.spawn(self._wait)
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _blocking_wait(self):
WaitForSingleObject(self._handle, INFINITE)
self.returncode = GetExitCodeProcess(self._handle)
return self.returncode
def _wait(self):
self.threadpool.spawn(self._blocking_wait).rawlink(self.result)
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
if self.returncode is None:
if not self._waiting:
self._waiting = True
self._wait()
result = self.result.wait(timeout=timeout)
if PY3 and timeout is not None and not self.result.ready():
raise TimeoutExpired(self.args, timeout)
return result
def send_signal(self, sig):
"""Send a signal to the process
"""
if sig == signal.SIGTERM:
self.terminate()
elif sig == signal.CTRL_C_EVENT:
os.kill(self.pid, signal.CTRL_C_EVENT)
elif sig == signal.CTRL_BREAK_EVENT:
os.kill(self.pid, signal.CTRL_BREAK_EVENT)
else:
raise ValueError("Unsupported signal: {}".format(sig))
def terminate(self):
"""Terminates the process
"""
TerminateProcess(self._handle, 1)
kill = terminate
else:
#
# POSIX methods
#
def rawlink(self, callback):
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
try:
DEVNULL
except NameError:
_devnull = object()
else:
_devnull = DEVNULL
if stdin is None:
pass
elif stdin == PIPE:
p2cread, p2cwrite = self.pipe_cloexec()
elif stdin == _devnull:
p2cread = self._get_devnull()
elif isinstance(stdin, int):
p2cread = stdin
else:
# Assuming file-like object
p2cread = stdin.fileno()
if stdout is None:
pass
elif stdout == PIPE:
c2pread, c2pwrite = self.pipe_cloexec()
elif stdout == _devnull:
c2pwrite = self._get_devnull()
elif isinstance(stdout, int):
c2pwrite = stdout
else:
# Assuming file-like object
c2pwrite = stdout.fileno()
if stderr is None:
pass
elif stderr == PIPE:
errread, errwrite = self.pipe_cloexec()
elif stderr == STDOUT:
errwrite = c2pwrite
elif stderr == _devnull:
errwrite = self._get_devnull()
elif isinstance(stderr, int):
errwrite = stderr
else:
# Assuming file-like object
errwrite = stderr.fileno()
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _set_cloexec_flag(self, fd, cloexec=True):
try:
cloexec_flag = fcntl.FD_CLOEXEC
except AttributeError:
cloexec_flag = 1
old = fcntl.fcntl(fd, fcntl.F_GETFD)
if cloexec:
fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
else:
fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag)
def _remove_nonblock_flag(self, fd):
flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK)
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def pipe_cloexec(self):
"""Create a pipe with FDs set CLOEXEC."""
# Pipes' FDs are set CLOEXEC by default because we don't want them
# to be inherited by other subprocesses: the CLOEXEC flag is removed
# from the child's FDs by _dup2(), between fork() and exec().
# This is not atomic: we would need the pipe2() syscall for that.
r, w = os.pipe()
self._set_cloexec_flag(r)
self._set_cloexec_flag(w)
return r, w
def _close_fds(self, keep):
# `keep` is a set of fds, so we
# use os.closerange from 3 to min(keep)
# and then from max(keep + 1) to MAXFD and
# loop through filling in the gaps.
# Under new python versions, we need to explicitly set
# passed fds to be inheritable or they will go away on exec
if hasattr(os, 'set_inheritable'):
set_inheritable = os.set_inheritable
else:
set_inheritable = lambda i, v: True
if hasattr(os, 'closerange'):
keep = sorted(keep)
min_keep = min(keep)
max_keep = max(keep)
os.closerange(3, min_keep)
os.closerange(max_keep + 1, MAXFD)
for i in xrange(min_keep, max_keep):
if i in keep:
set_inheritable(i, True)
continue
try:
os.close(i)
except:
pass
else:
for i in xrange(3, MAXFD):
if i in keep:
set_inheritable(i, True)
continue
try:
os.close(i)
except:
pass
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite,
restore_signals, start_new_session):
"""Execute program (POSIX version)"""
if PY3 and isinstance(args, (str, bytes)):
args = [args]
elif not PY3 and isinstance(args, string_types):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
self._loop.install_sigchld()
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
errpipe_read, errpipe_write = self.pipe_cloexec()
# errpipe_write must not be in the standard io 0, 1, or 2 fd range.
low_fds_to_close = []
while errpipe_write < 3:
low_fds_to_close.append(errpipe_write)
errpipe_write = os.dup(errpipe_write)
for low_fd in low_fds_to_close:
os.close(low_fd)
try:
try:
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = fork_and_watch(self._on_child, self._loop, True, fork)
except:
if gc_was_enabled:
gc.enable()
raise
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# When duping fds, if there arises a situation
# where one of the fds is either 0, 1 or 2, it
# is possible that it is overwritten (#12607).
if c2pwrite == 0:
c2pwrite = os.dup(c2pwrite)
if errwrite == 0 or errwrite == 1:
errwrite = os.dup(errwrite)
# Dup fds for child
def _dup2(a, b):
# dup2() removes the CLOEXEC flag but
# we must do it ourselves if dup2()
# would be a no-op (issue #10806).
if a == b:
self._set_cloexec_flag(a, False)
elif a is not None:
os.dup2(a, b)
self._remove_nonblock_flag(b)
_dup2(p2cread, 0)
_dup2(c2pwrite, 1)
_dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the
# same fd more than once, or standard fds.
closed = set([None])
for fd in [p2cread, c2pwrite, errwrite]:
if fd not in closed and fd > 2:
os.close(fd)
closed.add(fd)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
# Close all other fds, if asked for. This must be done
# after preexec_fn runs.
if close_fds:
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self._close_fds(fds_to_keep)
elif hasattr(os, 'get_inheritable'):
# close_fds was false, and we're on
# Python 3.4 or newer, so "all file
# descriptors except standard streams
# are closed, and inheritable handles
# are only inherited if the close_fds
# parameter is False."
for i in xrange(3, MAXFD):
try:
if i == errpipe_write or os.get_inheritable(i):
continue
os.close(i)
except:
pass
if restore_signals:
# restore the documented signals back to sig_dfl;
# not all will be defined on every platform
for sig in 'SIGPIPE', 'SIGXFZ', 'SIGXFSZ':
sig = getattr(signal, sig, None)
if sig is not None:
signal.signal(sig, signal.SIG_DFL)
if start_new_session:
os.setsid()
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info()
# Save the traceback and attach it to the exception object
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
finally:
# Make sure that the process exits no matter what.
# The return code does not matter much as it won't be
# reported to the application
os._exit(1)
# Parent
self._child_created = True
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
# self._devnull is not always defined.
devnull_fd = getattr(self, '_devnull', None)
if p2cread is not None and p2cwrite is not None and p2cread != devnull_fd:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None and c2pwrite != devnull_fd:
os.close(c2pwrite)
if errwrite is not None and errread is not None and errwrite != devnull_fd:
os.close(errwrite)
if devnull_fd is not None:
os.close(devnull_fd)
# Prevent a double close of these fds from __init__ on error.
self._closed_child_pipe_fds = True
# Wait for exec to fail or succeed; possibly raising exception
errpipe_read = FileObject(errpipe_read, 'rb')
data = errpipe_read.read()
finally:
if hasattr(errpipe_read, 'close'):
errpipe_read.close()
else:
os.close(errpipe_read)
if data != b"":
self.wait()
child_exception = pickle.loads(data)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
os.close(fd)
raise child_exception
def _handle_exitstatus(self, sts):
if os.WIFSIGNALED(sts):
self.returncode = -os.WTERMSIG(sts)
elif os.WIFEXITED(sts):
self.returncode = os.WEXITSTATUS(sts)
else:
# Should never happen
raise RuntimeError("Unknown child exit status!")
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if get_hub() is not getcurrent():
sig_pending = getattr(self._loop, 'sig_pending', True)
if sig_pending:
sleep(0.00001)
return self.returncode
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute.
:keyword timeout: The floating point number of seconds to wait.
Under Python 2, this is a gevent extension, and we simply return if it
expires. Under Python 3,
if this time elapses without finishing the process, :exc:`TimeoutExpired`
is raised."""
result = self.result.wait(timeout=timeout)
if PY3 and timeout is not None and not self.result.ready():
raise TimeoutExpired(self.args, timeout)
return result
def send_signal(self, sig):
"""Send a signal to the process
"""
os.kill(self.pid, sig)
def terminate(self):
"""Terminate the process with SIGTERM
"""
self.send_signal(signal.SIGTERM)
def kill(self):
"""Kill the process with SIGKILL
"""
self.send_signal(signal.SIGKILL)
class AlarmTask(Runnable):
""" Task to notify when a block is mined. """
def __init__(self, chain):
super().__init__()
self.callbacks = list()
self.chain = chain
self.chain_id = None
self.known_block_number = None
# TODO: Start with a larger sleep_time and decrease it as the
# probability of a new block increases.
self.sleep_time = 0.5
def start(self):
log.debug('Alarm task started', node=pex(self.chain.node_address))
self._stop_event = AsyncResult()
super().start()
def _run(self): # pylint: disable=method-hidden
try:
self.loop_until_stop()
finally:
self.callbacks = list()
def register_callback(self, callback):
""" Register a new callback.
Note:
The callback will be executed in the AlarmTask context and for
this reason it should not block, otherwise we can miss block
changes.
"""
if not callable(callback):
raise ValueError('callback is not a callable')
self.callbacks.append(callback)
def remove_callback(self, callback):
"""Remove callback from the list of callbacks if it exists"""
if callback in self.callbacks:
self.callbacks.remove(callback)
def loop_until_stop(self):
# The AlarmTask must have completed its first_run() before starting
# the background greenlet.
#
# This is required because the first run will synchronize the node with
# the blockchain since the last run.
assert self.chain_id, 'chain_id not set'
assert self.known_block_number is not None, 'known_block_number not set'
sleep_time = self.sleep_time
while self._stop_event.wait(sleep_time) is not True:
try:
latest_block = self.chain.get_block(block_identifier='latest')
except JSONDecodeError as e:
raise EthNodeCommunicationError(str(e))
self._maybe_run_callbacks(latest_block)
def first_run(self, known_block_number):
""" Blocking call to update the local state, if necessary. """
assert self.callbacks, 'callbacks not set'
latest_block = self.chain.get_block(block_identifier='latest')
log.debug(
'Alarm task first run',
known_block_number=known_block_number,
latest_block_number=latest_block['number'],
latest_gas_limit=latest_block['gasLimit'],
latest_block_hash=to_hex(latest_block['hash']),
)
self.known_block_number = known_block_number
self.chain_id = self.chain.network_id
self._maybe_run_callbacks(latest_block)
def _maybe_run_callbacks(self, latest_block):
""" Run the callbacks if there is at least one new block.
The callbacks are executed only if there is a new block, otherwise the
filters may try to poll for an inexisting block number and the Ethereum
client can return an JSON-RPC error.
"""
assert self.known_block_number is not None, 'known_block_number not set'
latest_block_number = latest_block['number']
missed_blocks = latest_block_number - self.known_block_number
if missed_blocks < 0:
log.critical(
'Block number decreased',
chain_id=self.chain_id,
known_block_number=self.known_block_number,
old_block_number=latest_block['number'],
old_gas_limit=latest_block['gasLimit'],
old_block_hash=to_hex(latest_block['hash']),
)
elif missed_blocks > 0:
log_details = dict(
known_block_number=self.known_block_number,
latest_block_number=latest_block_number,
latest_block_hash=to_hex(latest_block['hash']),
latest_block_gas_limit=latest_block['gasLimit'],
)
if missed_blocks > 1:
log_details['num_missed_blocks'] = missed_blocks - 1
log.debug(
'Received new block',
**log_details,
)
remove = list()
for callback in self.callbacks:
result = callback(latest_block)
if result is REMOVE_CALLBACK:
remove.append(callback)
for callback in remove:
self.callbacks.remove(callback)
self.known_block_number = latest_block_number
def stop(self):
self._stop_event.set(True)
log.debug('Alarm task stopped', node=pex(self.chain.node_address))
result = self.join()
# Callbacks should be cleaned after join
self.callbacks = []
return result
class Popen(object):
# copy of gevent.subprocess.Popen; won't be needed once gevent's subprocess supports pass_fds
def __init__(self, args, pass_fds=()):
hub = get_hub()
self._loop = hub.loop
self.pid = None
self.returncode = None
self.result = AsyncResult()
self._execute_child(args, pass_fds)
def __repr__(self):
return "<%s at 0x%x pid=%r returncode=%r>" % (self.__class__.__name__, id(self), self.pid, self.returncode)
def _on_child(self, watcher):
watcher.stop()
status = watcher.rstatus
if os.WIFSIGNALED(status):
self.returncode = -os.WTERMSIG(status)
else:
self.returncode = os.WEXITSTATUS(status)
self.result.set(self.returncode)
def rawlink(self, callback):
self.result.rawlink(linkproxy(callback, self))
def _execute_child(self, args, pass_fds):
executable = args[0]
self._loop.install_sigchld()
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = fork()
except:
if gc_was_enabled:
gc.enable()
raise
if self.pid == 0:
_close_fds(pass_fds)
try:
os.execvp(executable, args)
finally:
os._exit(1)
# Parent
self._watcher = self._loop.child(self.pid)
self._watcher.start(self._on_child, self._watcher)
if gc_was_enabled:
gc.enable()
def poll(self):
if self.returncode is None:
if get_hub() is not getcurrent():
sig_pending = getattr(self._loop, "sig_pending", True)
if sig_pending:
sleep(0.00001)
return self.returncode
def wait(self, timeout=None):
return self.result.wait(timeout=timeout)
def send_signal(self, sig):
os.kill(self.pid, sig)
class AlarmTask(Task):
""" Task to notify when a block is mined. """
def __init__(self, chain):
super(AlarmTask, self).__init__()
self.callbacks = list()
self.stop_event = AsyncResult()
self.chain = chain
self.last_block_number = None
# TODO: Start with a larger wait_time and decrease it as the
# probability of a new block increases.
self.wait_time = 0.5
self.last_loop = time.time()
def register_callback(self, callback):
""" Register a new callback.
Note:
The callback will be executed in the AlarmTask context and for
this reason it should not block, otherwise we can miss block
changes.
"""
if not callable(callback):
raise ValueError('callback is not a callable')
self.callbacks.append(callback)
def remove_callback(self, callback):
"""Remove callback from the list of callbacks if it exists"""
if callback in self.callbacks:
self.callbacks.remove(callback)
def _run(self): # pylint: disable=method-hidden
log.debug('starting block number', block_number=self.last_block_number)
sleep_time = 0
while self.stop_event.wait(sleep_time) is not True:
try:
self.poll_for_new_block()
except RaidenShuttingDown:
break
# we want this task to iterate in the tick of `wait_time`, so take
# into account how long we spent executing one tick.
self.last_loop = time.time()
work_time = self.last_loop - self.last_loop
if work_time > self.wait_time:
log.warning(
'alarm loop is taking longer than the wait time',
work_time=work_time,
wait_time=self.wait_time,
)
sleep_time = 0.001
else:
sleep_time = self.wait_time - work_time
# stopping
self.callbacks = list()
def poll_for_new_block(self):
current_block = self.chain.block_number()
if current_block > self.last_block_number + 1:
difference = current_block - self.last_block_number - 1
log.error(
'alarm missed %s blocks',
difference,
)
if current_block != self.last_block_number:
log.debug(
'new block',
number=current_block,
timestamp=self.last_loop,
)
self.last_block_number = current_block
remove = list()
for callback in self.callbacks:
try:
result = callback(current_block)
except RaidenShuttingDown:
break
except: # pylint: disable=bare-except # noqa
log.exception('unexpected exception on alarm')
else:
if result is REMOVE_CALLBACK:
remove.append(callback)
for callback in remove:
self.callbacks.remove(callback)
def start(self):
self.last_block_number = self.chain.block_number()
super(AlarmTask, self).start()
def stop_and_wait(self):
self.stop_event.set(True)
gevent.wait(self)
def stop_async(self):
self.stop_event.set(True)
class Popen(object):
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0, threadpool=None):
"""Create new Popen instance."""
if not isinstance(bufsize, integer_types):
raise TypeError("bufsize must be an integer")
hub = get_hub()
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None or
stderr is not None):
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
if threadpool is None:
threadpool = hub.threadpool
self.threadpool = threadpool
self._waiting = False
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
assert threadpool is None
self._loop = hub.loop
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
self.result = AsyncResult()
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr)
self._execute_child(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
self.stdin = FileObject(p2cwrite, 'wb')
if c2pread is not None:
if universal_newlines:
self.stdout = FileObject(c2pread, 'rU')
else:
self.stdout = FileObject(c2pread, 'rb')
if errread is not None:
if universal_newlines:
self.stderr = FileObject(errread, 'rU')
else:
self.stderr = FileObject(errread, 'rb')
def __repr__(self):
return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode)
def _on_child(self, watcher):
watcher.stop()
status = watcher.rstatus
if os.WIFSIGNALED(status):
self.returncode = -os.WTERMSIG(status)
else:
self.returncode = os.WEXITSTATUS(status)
self.result.set(self.returncode)
def communicate(self, input=None):
"""Interact with process: Send data to stdin. Read data from
stdout and stderr, until end-of-file is reached. Wait for
process to terminate. The optional input argument should be a
string to be sent to the child process, or None, if no data
should be sent to the child.
communicate() returns a tuple (stdout, stderr)."""
greenlets = []
if self.stdin:
greenlets.append(spawn(write_and_close, self.stdin, input))
if self.stdout:
stdout = spawn(self.stdout.read)
greenlets.append(stdout)
else:
stdout = None
if self.stderr:
stderr = spawn(self.stderr.read)
greenlets.append(stderr)
else:
stderr = None
joinall(greenlets)
if self.stdout:
self.stdout.close()
if self.stderr:
self.stderr.close()
self.wait()
return (None if stdout is None else stdout.value or '',
None if stderr is None else stderr.value or '')
def poll(self):
return self._internal_poll()
if mswindows:
#
# Windows methods
#
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin is None and stdout is None and stderr is None:
return (None, None, None, None, None, None)
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
if stdin is None:
p2cread = GetStdHandle(STD_INPUT_HANDLE)
if p2cread is None:
p2cread, _ = CreatePipe(None, 0)
elif stdin == PIPE:
p2cread, p2cwrite = CreatePipe(None, 0)
elif isinstance(stdin, int):
p2cread = msvcrt.get_osfhandle(stdin)
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout is None:
c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
if c2pwrite is None:
_, c2pwrite = CreatePipe(None, 0)
elif stdout == PIPE:
c2pread, c2pwrite = CreatePipe(None, 0)
elif isinstance(stdout, int):
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr is None:
errwrite = GetStdHandle(STD_ERROR_HANDLE)
if errwrite is None:
_, errwrite = CreatePipe(None, 0)
elif stderr == PIPE:
errread, errwrite = CreatePipe(None, 0)
elif stderr == STDOUT:
errwrite = c2pwrite
elif isinstance(stderr, int):
errwrite = msvcrt.get_osfhandle(stderr)
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _make_inheritable(self, handle):
"""Return a duplicate of handle, which is inheritable"""
return DuplicateHandle(GetCurrentProcess(),
handle, GetCurrentProcess(), 0, 1,
DUPLICATE_SAME_ACCESS)
def _find_w9xpopen(self):
"""Find and return absolut path to w9xpopen.exe"""
w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
# Eeek - file-not-found - possibly an embedding
# situation - see if we can locate it in sys.exec_prefix
w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
"w9xpopen.exe")
if not os.path.exists(w9xpopen):
raise RuntimeError("Cannot locate w9xpopen.exe, which is "
"needed for Popen to work with your "
"shell or platform.")
return w9xpopen
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Execute program (MS Windows version)"""
if not isinstance(args, string_types):
args = list2cmdline(args)
# Process startup details
if startupinfo is None:
startupinfo = STARTUPINFO()
if None not in (p2cread, c2pwrite, errwrite):
startupinfo.dwFlags |= STARTF_USESTDHANDLES
startupinfo.hStdInput = p2cread
startupinfo.hStdOutput = c2pwrite
startupinfo.hStdError = errwrite
if shell:
startupinfo.dwFlags |= STARTF_USESHOWWINDOW
startupinfo.wShowWindow = SW_HIDE
comspec = os.environ.get("COMSPEC", "cmd.exe")
args = '{} /c "{}"'.format(comspec, args)
if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com":
# Win9x, or using command.com on NT. We need to
# use the w9xpopen intermediate program. For more
# information, see KB Q150956
# (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
w9xpopen = self._find_w9xpopen()
args = '"%s" %s' % (w9xpopen, args)
# Not passing CREATE_NEW_CONSOLE has been known to
# cause random failures on win9x. Specifically a
# dialog: "Your program accessed mem currently in
# use at xxx" and a hopeful warning about the
# stability of your system. Cost is Ctrl+C wont
# kill children.
creationflags |= CREATE_NEW_CONSOLE
# Start the process
try:
hp, ht, pid, tid = CreateProcess(executable, args,
# no special security
None, None,
int(not close_fds),
creationflags,
env,
cwd,
startupinfo)
except pywintypes.error as e:
# Translate pywintypes.error to WindowsError, which is
# a subclass of OSError. FIXME: We should really
# translate errno using _sys_errlist (or similar), but
# how can this be done from Python?
raise WindowsError(*e.args)
finally:
# Child is launched. Close the parent's copy of those pipe
# handles that only the child should have open. You need
# to make sure that no handles to the write end of the
# output pipe are maintained in this process or else the
# pipe will not close when the child process exits and the
# ReadFile will hang.
if p2cread is not None:
p2cread.Close()
if c2pwrite is not None:
c2pwrite.Close()
if errwrite is not None:
errwrite.Close()
# Retain the process handle, but close the thread handle
self._handle = hp
self.pid = pid
ht.Close()
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
self.returncode = GetExitCodeProcess(self._handle)
self.result.set(self.returncode)
return self.returncode
def rawlink(self, callback):
if not self.result.ready() and not self._waiting:
self._waiting = True
Greenlet.spawn(self._wait)
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _blocking_wait(self):
WaitForSingleObject(self._handle, INFINITE)
self.returncode = GetExitCodeProcess(self._handle)
return self.returncode
def _wait(self):
self.threadpool.spawn(self._blocking_wait).rawlink(self.result)
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
if self.returncode is None:
if not self._waiting:
self._waiting = True
self._wait()
return self.result.wait(timeout=timeout)
def send_signal(self, sig):
"""Send a signal to the process
"""
if sig == signal.SIGTERM:
self.terminate()
elif sig == signal.CTRL_C_EVENT:
os.kill(self.pid, signal.CTRL_C_EVENT)
elif sig == signal.CTRL_BREAK_EVENT:
os.kill(self.pid, signal.CTRL_BREAK_EVENT)
else:
raise ValueError("Unsupported signal: {}".format(sig))
def terminate(self):
"""Terminates the process
"""
TerminateProcess(self._handle, 1)
kill = terminate
else:
#
# POSIX methods
#
def rawlink(self, callback):
self.result.rawlink(linkproxy(callback, self))
# XXX unlink
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
if stdin is None:
pass
elif stdin == PIPE:
p2cread, p2cwrite = self.pipe_cloexec()
elif isinstance(stdin, int):
p2cread = stdin
else:
# Assuming file-like object
p2cread = stdin.fileno()
if stdout is None:
pass
elif stdout == PIPE:
c2pread, c2pwrite = self.pipe_cloexec()
elif isinstance(stdout, int):
c2pwrite = stdout
else:
# Assuming file-like object
c2pwrite = stdout.fileno()
if stderr is None:
pass
elif stderr == PIPE:
errread, errwrite = self.pipe_cloexec()
elif stderr == STDOUT:
errwrite = c2pwrite
elif isinstance(stderr, int):
errwrite = stderr
else:
# Assuming file-like object
errwrite = stderr.fileno()
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def _set_cloexec_flag(self, fd, cloexec=True):
try:
cloexec_flag = fcntl.FD_CLOEXEC
except AttributeError:
cloexec_flag = 1
old = fcntl.fcntl(fd, fcntl.F_GETFD)
if cloexec:
fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
else:
fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag)
def _remove_nonblock_flag(self, fd):
flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK)
fcntl.fcntl(fd, fcntl.F_SETFL, flags)
def pipe_cloexec(self):
"""Create a pipe with FDs set CLOEXEC."""
# Pipes' FDs are set CLOEXEC by default because we don't want them
# to be inherited by other subprocesses: the CLOEXEC flag is removed
# from the child's FDs by _dup2(), between fork() and exec().
# This is not atomic: we would need the pipe2() syscall for that.
r, w = os.pipe()
self._set_cloexec_flag(r)
self._set_cloexec_flag(w)
return r, w
def _close_fds(self, but):
if hasattr(os, 'closerange'):
os.closerange(3, but)
os.closerange(but + 1, MAXFD)
else:
for i in xrange(3, MAXFD):
if i == but:
continue
try:
os.close(i)
except:
pass
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Execute program (POSIX version)"""
if isinstance(args, string_types):
args = [args]
else:
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
if executable:
args[0] = executable
if executable is None:
executable = args[0]
self._loop.install_sigchld()
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
errpipe_read, errpipe_write = self.pipe_cloexec()
try:
try:
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = fork()
except:
if gc_was_enabled:
gc.enable()
raise
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# When duping fds, if there arises a situation
# where one of the fds is either 0, 1 or 2, it
# is possible that it is overwritten (#12607).
if c2pwrite == 0:
c2pwrite = os.dup(c2pwrite)
if errwrite == 0 or errwrite == 1:
errwrite = os.dup(errwrite)
# Dup fds for child
def _dup2(a, b):
# dup2() removes the CLOEXEC flag but
# we must do it ourselves if dup2()
# would be a no-op (issue #10806).
if a == b:
self._set_cloexec_flag(a, False)
elif a is not None:
os.dup2(a, b)
self._remove_nonblock_flag(b)
_dup2(p2cread, 0)
_dup2(c2pwrite, 1)
_dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the
# same fd more than once, or standard fds.
closed = set([None])
for fd in [p2cread, c2pwrite, errwrite]:
if fd not in closed and fd > 2:
os.close(fd)
closed.add(fd)
# Close all other fds, if asked for
if close_fds:
self._close_fds(but=errpipe_write)
if cwd is not None:
os.chdir(cwd)
if preexec_fn:
preexec_fn()
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info()
# Save the traceback and attach it to the exception object
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb)
exc_value.child_traceback = ''.join(exc_lines)
os.write(errpipe_write, pickle.dumps(exc_value))
finally:
# Make sure that the process exits no matter what.
# The return code does not matter much as it won't be
# reported to the application
os._exit(1)
# Parent
self._watcher = self._loop.child(self.pid)
self._watcher.start(self._on_child, self._watcher)
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what
os.close(errpipe_write)
if p2cread is not None and p2cwrite is not None:
os.close(p2cread)
if c2pwrite is not None and c2pread is not None:
os.close(c2pwrite)
if errwrite is not None and errread is not None:
os.close(errwrite)
# Wait for exec to fail or succeed; possibly raising exception
errpipe_read = FileObject(errpipe_read, 'rb')
data = errpipe_read.read()
finally:
if hasattr(errpipe_read, 'close'):
errpipe_read.close()
else:
os.close(errpipe_read)
if data != b"":
self.wait()
child_exception = pickle.loads(data)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
os.close(fd)
raise child_exception
def _handle_exitstatus(self, sts):
if os.WIFSIGNALED(sts):
self.returncode = -os.WTERMSIG(sts)
elif os.WIFEXITED(sts):
self.returncode = os.WEXITSTATUS(sts)
else:
# Should never happen
raise RuntimeError("Unknown child exit status!")
def _internal_poll(self):
"""Check if child process has terminated. Returns returncode
attribute.
"""
if self.returncode is None:
if get_hub() is not getcurrent():
sig_pending = getattr(self._loop, 'sig_pending', True)
if sig_pending:
sleep(0.00001)
return self.returncode
def wait(self, timeout=None):
"""Wait for child process to terminate. Returns returncode
attribute."""
return self.result.wait(timeout=timeout)
def send_signal(self, sig):
"""Send a signal to the process
"""
os.kill(self.pid, sig)
def terminate(self):
"""Terminate the process with SIGTERM
"""
self.send_signal(signal.SIGTERM)
def kill(self):
"""Kill the process with SIGKILL
"""
self.send_signal(signal.SIGKILL)
class ThreadManager(object):
"""
@brief Manage spawning threads of multiple kinds and ensure they're alive.
TODO: Add heartbeats with zeromq for monitoring and restarting.
"""
def __init__(self, heartbeat_secs=10.0, failure_notify_callback=None):
"""
Creates a ThreadManager.
@param heartbeat_secs Seconds between heartbeats.
@param failure_notify_callback Callback to execute when a child fails unexpectedly. Should be
a callable taking two params: this process supervisor, and the
thread that failed.
"""
super(ThreadManager, self).__init__()
# NOTE: Assumes that pids never overlap between the various process types
self.children = []
self.heartbeat_secs = heartbeat_secs
self._shutting_down = False
self._failure_notify_callback = failure_notify_callback
self._shutdown_event = AsyncResult()
def _create_thread(self, target=None, **kwargs):
"""
Creates a "thread" of the proper type.
"""
return PyonThread(target=target, **kwargs)
def spawn(self, target=None, **kwargs):
"""
@brief Spawn a pyon thread
"""
log.debug("ThreadManager.spawn, target=%s, kwargs=%s", target, kwargs)
proc = self._create_thread(target=target, **kwargs)
proc.supervisor = self
proc.start()
self.children.append(proc)
# install failure monitor
proc.proc.link_exception(self._child_failed)
return proc
def _child_failed(self, gproc):
# extract any PyonThreadTracebacks - one should be last
extra = ""
if len(gproc.exception.args) and isinstance(gproc.exception.args[-1], PyonThreadTraceback):
extra = "\n" + str(gproc.exception.args[-1])
log.error("Child failed with an exception: (%s) %s%s", gproc, gproc.exception, extra)
if self._failure_notify_callback:
self._failure_notify_callback(gproc)
def ensure_ready(self, proc, errmsg=None, timeout=10):
"""
Waits until either the thread dies or reports it is ready, whichever comes first.
If the thread dies or times out while waiting for it to be ready, a ContainerError is raised.
You must be sure the thread implements get_ready_event properly, otherwise this method
returns immediately as the base class behavior simply passes.
@param proc The thread to wait on.
@param errmsg A custom error message to put in the ContainerError's message. May be blank.
@param timeout Amount of time (in seconds) to wait for the ready, default 10 seconds.
@throws ContainerError If the thread dies or if we get a timeout before the process signals ready.
"""
if not errmsg:
errmsg = "ensure_ready failed"
ev = Event()
def cb(*args, **kwargs):
ev.set()
# link either a greenlet failure due to exception OR a success via ready event
proc.proc.link_exception(cb)
proc.get_ready_event().rawlink(cb)
retval = ev.wait(timeout=timeout)
# unlink the events: ready event is probably harmless but the exception one, we want to install our own later
proc.get_ready_event().unlink(cb)
# if the thread is stopped while we are waiting, proc.proc is set to None
if proc.proc is not None:
proc.proc.unlink(cb)
# raise an exception if:
# - we timed out
# - we caught an exception
if not retval:
raise ContainerError("%s (timed out)" % errmsg)
elif proc.proc is not None and proc.proc.dead and not proc.proc.successful():
raise ContainerError("%s (failed): %s" % (errmsg, proc.proc.exception))
def child_stopped(self, proc):
if proc in self.children:
# no longer need to listen for exceptions
if proc.proc is not None:
proc.proc.unlink(self._child_failed)
def join_children(self, timeout=None):
""" Give child threads "timeout" seconds to shutdown, then forcibly terminate. """
time_start = time.time()
child_count = len(self.children)
for proc in self.children:
# if a child thread has already exited, we don't need to wait on anything -
# it's already good to go and can be considered joined. Otherwise we will likely
# double call notify_stop which is a bad thing.
if proc.proc.dead:
continue
time_elapsed = time.time() - time_start
if timeout is not None:
time_remaining = timeout - time_elapsed
if time_remaining > 0:
# The nice way; let it do cleanup
try:
proc.notify_stop()
proc.join(time_remaining)
except Exception:
# not playing nice? just kill it.
proc.stop()
else:
# Out of time. Cya, sucker
proc.stop()
else:
proc.join()
time_elapsed = time.time() - time_start
log.debug('Took %.2fs to shutdown %d child threads', time_elapsed, child_count)
return time_elapsed
def wait_children(self, timeout=None):
"""
Performs a join to allow children to complete, then a get() to fetch their results.
This will raise an exception if any of the children raises an exception.
"""
self.join_children(timeout=timeout)
return [x.get() for x in self.children]
def target(self):
try:
while not self._shutting_down:
self.send_heartbeats()
self._shutdown_event.wait(timeout=self.heartbeat_secs)
except:
log.error('thread died', exc_info=True)
def send_heartbeats(self):
""" TODO: implement heartbeat and monitors """
#log.debug('lub-dub')
pass
def shutdown(self, timeout=30.0):
"""
@brief Give child thread "timeout" seconds to shutdown, then forcibly terminate.
"""
self._shutting_down = True
self._shutdown_event.set(True)
unset = shutdown_or_die(timeout) # Failsafe in case the following doesn't work
elapsed = self.join_children(timeout)
#self.stop()
unset()
return elapsed
class Popen(object):
# copy of gevent.subprocess.Popen; won't be needed once gevent's subprocess supports pass_fds
def __init__(self, args, pass_fds=()):
hub = get_hub()
self._loop = hub.loop
self.pid = None
self.returncode = None
self.result = AsyncResult()
self._execute_child(args, pass_fds)
def __repr__(self):
return '<%s at 0x%x pid=%r returncode=%r>' % (
self.__class__.__name__, id(self), self.pid, self.returncode)
def _on_child(self, watcher):
watcher.stop()
status = watcher.rstatus
if os.WIFSIGNALED(status):
self.returncode = -os.WTERMSIG(status)
else:
self.returncode = os.WEXITSTATUS(status)
self.result.set(self.returncode)
def rawlink(self, callback):
self.result.rawlink(linkproxy(callback, self))
def _execute_child(self, args, pass_fds):
executable = args[0]
self._loop.install_sigchld()
gc_was_enabled = gc.isenabled()
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
gc.disable()
try:
self.pid = fork()
except:
if gc_was_enabled:
gc.enable()
raise
if self.pid == 0:
_close_fds(pass_fds)
try:
os.execvp(executable, args)
finally:
os._exit(1)
# Parent
self._watcher = self._loop.child(self.pid)
self._watcher.start(self._on_child, self._watcher)
if gc_was_enabled:
gc.enable()
def poll(self):
if self.returncode is None:
if get_hub() is not getcurrent():
sig_pending = getattr(self._loop, 'sig_pending', True)
if sig_pending:
sleep(0.00001)
return self.returncode
def wait(self, timeout=None):
return self.result.wait(timeout=timeout)
def send_signal(self, sig):
os.kill(self.pid, sig)
class StartMediatedTransferTask(Task):
def __init__(self, transfermanager, amount, target, done_result):
super(StartMediatedTransferTask, self).__init__()
self.amount = amount
self.address = transfermanager.assetmanager.raiden.address
self.target = target
self.transfermanager = transfermanager
self.done_result = done_result
def __repr__(self):
return '<{} {}>'.format(
self.__class__.__name__,
pex(self.address),
)
def _run(self): # pylint: disable=method-hidden,too-many-locals
amount = self.amount
target = self.target
raiden = self.transfermanager.assetmanager.raiden
fee = 0
# there are no guarantees that the next_hop will follow the same route
routes = self.transfermanager.assetmanager.get_best_routes(
amount,
target,
lock_timeout=None,
)
if log.isEnabledFor(logging.DEBUG):
log.debug(
'START MEDIATED TRANSFER initiator:%s target:%s',
pex(self.address),
pex(self.target),
)
for path, forward_channel in routes:
# try a new secret
secret = sha3(hex(random.getrandbits(256)))
hashlock = sha3(secret)
next_hop = path[1]
if log.isEnabledFor(logging.DEBUG):
log.debug(
'START MEDIATED TRANSFER NEW PATH path:%s hashlock:%s',
lpex(path),
pex(hashlock),
)
self.transfermanager.register_task_for_hashlock(self, hashlock)
lock_expiration = (raiden.chain.block_number() +
forward_channel.settle_timeout -
raiden.config['reveal_timeout'])
mediated_transfer = forward_channel.create_mediatedtransfer(
raiden.address,
target,
fee,
amount,
lock_expiration,
hashlock,
)
raiden.sign(mediated_transfer)
forward_channel.register_transfer(mediated_transfer)
response = self.send_and_wait_valid(raiden, path,
mediated_transfer)
# `next_hop` timedout
if response is None:
self.transfermanager.on_hashlock_result(hashlock, False)
# someone down the line timedout / couldn't proceed
elif isinstance(response, (RefundTransfer, TransferTimeout)):
self.transfermanager.on_hashlock_result(hashlock, False)
# `target` received the MediatedTransfer
elif response.sender == target and isinstance(
response, SecretRequest):
secret_message = Secret(secret)
raiden.sign(secret_message)
raiden.send_async(target, secret_message)
# register the secret now and just incur with the additional
# overhead of retrying until the `next_hop` receives the secret
# forward_channel.register_secret(secret)
# wait until `next_hop` received the secret to syncronize our
# state (otherwise we can send a new transfer with an invalid
# locksroot while the secret is in transit that will incur into
# additional retry/timeout latency)
next_hop = path[1]
while True:
response = self.response_message.wait()
# critical write section
self.response_message = AsyncResult()
# /critical write section
if isinstance(response,
Secret) and response.sender == next_hop:
# critical read/write section
# The channel and it's queue must be locked, a transfer
# must not be created while we update the balance_proof.
forward_channel.claim_lock(secret)
raiden.send_async(next_hop, secret_message)
# /critical write section
self.transfermanager.on_hashlock_result(hashlock, True)
self.done_result.set(True)
return
log.error(
'Invalid message ignoring. %s',
repr(response),
)
else:
log.error(
'Unexpected response %s',
repr(response),
)
self.transfermanager.on_hashlock_result(hashlock, False)
if log.isEnabledFor(logging.DEBUG):
log.debug(
'START MEDIATED TRANSFER FAILED initiator:%s target:%s',
pex(self.address),
pex(self.target),
)
self.done_result.set(False) # all paths failed
def send_and_wait_valid(self, raiden, path, mediated_transfer): # pylint: disable=no-self-use
""" Send the `mediated_transfer` and wait for either a message from
`target` or the `next_hop`.
Validate the message received and discards the invalid ones. The most
important case being next_hop sending a SecretRequest.
"""
message_timeout = raiden.config['msg_timeout']
next_hop = path[1]
target = path[-1]
current_time = time.time()
limit_time = current_time + message_timeout
# this event is used by the transfermanager to notify the task that a
# response was received
self.response_message = AsyncResult()
raiden.send_async(next_hop, mediated_transfer)
while current_time <= limit_time:
# wait for a response message (not the Ack for the transfer)
response = self.response_message.wait(limit_time - current_time)
# reset so that a value can be received either because the current
# result was invalid or because we will wait for the next message.
#
# critical write section
self.response_message = AsyncResult()
# /critical write section
if response is None:
if log.isEnabledFor(logging.DEBUG):
log.debug(
'MEDIATED TRANSFER TIMED OUT hashlock:%s',
pex(mediated_transfer.lock.hashlock),
)
return None
if response.sender == next_hop:
if isinstance(response, (RefundTransfer, TransferTimeout)):
return response
else:
if log.isEnabledFor(logging.INFO):
log.info(
'Partner %s sent an invalid message',
pex(next_hop),
)
return None
if response.sender == target:
if isinstance(response, SecretRequest):
return response
else:
if log.isEnabledFor(logging.INFO):
log.info(
'target %s sent an invalid message',
pex(target),
)
return None
current_time = time.time()
if log.isEnabledFor(logging.ERROR):
log.error(
'Invalid message ignoring. %s',
repr(response),
)
return None
class AlarmTask(Runnable):
""" Task to notify when a block is mined. """
def __init__(self, chain):
super().__init__()
self.callbacks = list()
self.chain = chain
self.chain_id = None
self.known_block_number = None
self._stop_event = AsyncResult()
# TODO: Start with a larger sleep_time and decrease it as the
# probability of a new block increases.
self.sleep_time = 0.5
def start(self):
log.debug('Alarm task started', node=pex(self.chain.node_address))
self._stop_event.set(False)
super().start()
def _run(self): # pylint: disable=method-hidden
try:
self.loop_until_stop()
finally:
self.callbacks = list()
def register_callback(self, callback):
""" Register a new callback.
Note:
The callback will be executed in the AlarmTask context and for
this reason it should not block, otherwise we can miss block
changes.
"""
if not callable(callback):
raise ValueError('callback is not a callable')
self.callbacks.append(callback)
def remove_callback(self, callback):
"""Remove callback from the list of callbacks if it exists"""
if callback in self.callbacks:
self.callbacks.remove(callback)
def loop_until_stop(self):
# The AlarmTask must have completed its first_run() before starting
# the background greenlet.
#
# This is required because the first run will synchronize the node with
# the blockchain since the last run.
assert self.chain_id, 'chain_id not set'
assert self.known_block_number is not None, 'known_block_number not set'
chain_id = self.chain_id
sleep_time = self.sleep_time
while self._stop_event.wait(sleep_time) is not True:
try:
latest_block = self.chain.get_block(block_identifier='latest')
except JSONDecodeError as e:
raise EthNodeCommunicationError(str(e))
self._maybe_run_callbacks(latest_block)
if chain_id != self.chain.network_id:
raise RuntimeError(
'Changing the underlying blockchain while the Raiden node is running '
'is not supported.',
)
def first_run(self, known_block_number):
""" Blocking call to update the local state, if necessary. """
assert self.callbacks, 'callbacks not set'
chain_id = self.chain.network_id
latest_block = self.chain.get_block(block_identifier='latest')
log.debug(
'Alarm task first run',
known_block_number=known_block_number,
latest_block_number=latest_block['number'],
latest_gas_limit=latest_block['gasLimit'],
latest_block_hash=to_hex(latest_block['hash']),
)
self.known_block_number = known_block_number
self.chain_id = chain_id
self._maybe_run_callbacks(latest_block)
def _maybe_run_callbacks(self, latest_block):
""" Run the callbacks if there is at least one new block.
The callbacks are executed only if there is a new block, otherwise the
filters may try to poll for an inexisting block number and the Ethereum
client can return an JSON-RPC error.
"""
assert self.known_block_number is not None, 'known_block_number not set'
latest_block_number = latest_block['number']
missed_blocks = latest_block_number - self.known_block_number
if missed_blocks < 0:
log.critical(
'Block number decreased',
chain_id=self.chain_id,
known_block_number=self.known_block_number,
old_block_number=latest_block['number'],
old_gas_limit=latest_block['gasLimit'],
old_block_hash=to_hex(latest_block['hash']),
)
elif missed_blocks > 0:
log_details = dict(
known_block_number=self.known_block_number,
latest_block_number=latest_block_number,
latest_block_hash=to_hex(latest_block['hash']),
latest_block_gas_limit=latest_block['gasLimit'],
)
if missed_blocks > 1:
log_details['num_missed_blocks'] = missed_blocks - 1
log.debug(
'Received new block',
**log_details,
)
remove = list()
for callback in self.callbacks:
result = callback(latest_block)
if result is REMOVE_CALLBACK:
remove.append(callback)
for callback in remove:
self.callbacks.remove(callback)
self.known_block_number = latest_block_number
def stop(self):
self._stop_event.set(True)
log.debug('Alarm task stopped', node=pex(self.chain.node_address))
result = self.join()
# Callbacks should be cleaned after join
self.callbacks = []
return result
class EndMediatedTransferTask(Task):
""" Task that request a secret for a registered transfer. """
def __init__(self, transfermanager, originating_transfer):
super(EndMediatedTransferTask, self).__init__()
self.address = transfermanager.assetmanager.raiden.address
self.transfermanager = transfermanager
self.originating_transfer = originating_transfer
hashlock = originating_transfer.lock.hashlock
self.transfermanager.register_task_for_hashlock(self, hashlock)
def __repr__(self):
return '<{} {}>'.format(
self.__class__.__name__,
pex(self.address),
)
def _run(self): # pylint: disable=method-hidden
mediated_transfer = self.originating_transfer
assetmanager = self.transfermanager.assetmanager
originating_channel = assetmanager.get_channel_by_partner_address(
mediated_transfer.sender)
raiden = assetmanager.raiden
log.debug(
'END MEDIATED TRANSFER %s -> %s msghash:%s hashlock:%s',
pex(mediated_transfer.target),
pex(mediated_transfer.initiator),
pex(mediated_transfer.hash),
pex(mediated_transfer.lock.hashlock),
)
secret_request = SecretRequest(mediated_transfer.lock.hashlock)
raiden.sign(secret_request)
response = self.send_and_wait_valid(raiden, mediated_transfer,
secret_request)
if response is None:
timeout_message = originating_channel.create_timeouttransfer_for(
mediated_transfer)
raiden.send_async(mediated_transfer.sender, timeout_message)
self.transfermanager.on_hashlock_result(
mediated_transfer.lock.hashlock, False)
return
# register the secret so that a balance proof can be created but don't
# claim until our partner has informed us that it's internal state is
# updated
originating_channel.register_secret(response.secret)
secret_message = Secret(response.secret)
raiden.sign(secret_message)
raiden.send_async(mediated_transfer.sender, secret_message)
# wait for the secret from `sender` to claim the lock
while True:
response = self.response_message.wait()
# critical write section
self.response_message = AsyncResult()
# /critical write section
if isinstance(
response,
Secret) and response.sender == mediated_transfer.sender:
originating_channel.claim_lock(response.secret)
self.transfermanager.on_hashlock_result(
mediated_transfer.lock.hashlock, True)
return
def send_and_wait_valid(self, raiden, mediated_transfer, secret_request):
message_timeout = raiden.config['msg_timeout']
current_time = time.time()
limit_time = current_time + message_timeout
self.response_message = AsyncResult()
raiden.send_async(mediated_transfer.initiator, secret_request)
while current_time <= limit_time:
response = self.response_message.wait(limit_time - current_time)
# critical write section
self.response_message = AsyncResult()
# /critical write section
if response is None:
log.error(
'SECRETREQUEST TIMED OUT node:%s msghash:%s hashlock:%s',
pex(raiden.address),
pex(secret_request.hash),
pex(mediated_transfer.lock.hashlock),
)
return None
if isinstance(response, Secret):
if sha3(response.secret) != mediated_transfer.lock.hashlock:
log.error('Secret doesnt match the hashlock, ignoring.')
continue
return response
return None
def takes_too_long(self, noticear=None):
if noticear is not None:
noticear.set(True)
ar = AsyncResult()
ar.wait()
class AlarmTask(Runnable):
""" Task to notify when a block is mined. """
def __init__(self, chain):
super().__init__()
self.callbacks = list()
self.chain = chain
self.chain_id = None
self.last_block_number = None
self._stop_event = AsyncResult()
# TODO: Start with a larger sleep_time and decrease it as the
# probability of a new block increases.
self.sleep_time = 0.5
def _run(self): # pylint: disable=method-hidden
try:
self.loop_until_stop()
finally:
self.callbacks = list()
def register_callback(self, callback):
""" Register a new callback.
Note:
The callback will be executed in the AlarmTask context and for
this reason it should not block, otherwise we can miss block
changes.
"""
if not callable(callback):
raise ValueError('callback is not a callable')
self.callbacks.append(callback)
def remove_callback(self, callback):
"""Remove callback from the list of callbacks if it exists"""
if callback in self.callbacks:
self.callbacks.remove(callback)
def loop_until_stop(self):
# The AlarmTask must have completed its first_run() before starting
# the background greenlet.
#
# This is required because the first run will synchronize the node with
# the blockchain since the last run.
assert self.chain_id, 'chain_id not set'
assert self.last_block_number, 'last_block_number not set'
chain_id = self.chain_id
sleep_time = self.sleep_time
while self._stop_event.wait(sleep_time) is not True:
last_block_number = self.last_block_number
current_block = self.chain.block_number()
if chain_id != self.chain.network_id:
raise RuntimeError(
'Changing the underlying blockchain while the Raiden node is running '
'is not supported.', )
if current_block != last_block_number:
log.debug('new block', number=current_block)
if current_block > last_block_number + 1:
missed_blocks = current_block - last_block_number - 1
log.info(
'missed blocks',
missed_blocks=missed_blocks,
current_block=current_block,
)
self._run_callbacks(current_block)
def first_run(self):
# callbacks must be executed during the first run to update the node state
assert self.callbacks, 'callbacks not set'
chain_id = self.chain.network_id
current_block = self.chain.block_number()
log.debug('starting at block number', current_block=current_block)
self._run_callbacks(current_block)
self.chain_id = chain_id
def _run_callbacks(self, current_block):
remove = list()
for callback in self.callbacks:
result = callback(current_block)
if result is REMOVE_CALLBACK:
remove.append(callback)
for callback in remove:
self.callbacks.remove(callback)
self.last_block_number = current_block
def stop(self):
self._stop_event.set(True)
return self.join()
class TransferTask(Task):
"""
Normal Operation (Transfer A > C)
A: Initiator Creates Secret
A: MediatedTransfer > B
B: MediatedTransfer > C
C: SecretRequest > A (implicitly signs, that valid transfer was received)
A: Secret > C
C: Secret > B
Timeout (Transfer A > C)
A: Initiator Creates Secret
A: MediatedTransfer > B
B: MediatedTransfer > C
Failure: No Ack from C
B: TransferTimeout > A
Resolution: A won't reveal the secret, tries new transfer, B bans C
CancelTransfer (Transfer A > D)
A: Initiator Creates Secret
A: MediatedTransfer > B
B: MediatedTransfer > C
Failure: C can not establish path to D (e.g. insufficient distributable, no active node)
C: CancelTransfer > B (levels out balance)
B: MediatedTransfer > C2
C2: MediatedTransfer > D
...
"""
block_expiration = 120 # FIXME, this needs to timeout on block expiration
timeout_per_hop = 10
def __init__(self, transfermanager, amount, target, hashlock,
expiration=None, originating_transfer=None, secret=None): # fee!
assert isinstance(transfermanager, transfermanagermodule.TransferManager)
self.transfermanager = transfermanager
self.assetmanager = transfermanager.assetmanager
self.raiden = transfermanager.assetmanager.raiden
self.amount = amount
self.target = target
self.hashlock = hashlock
self.expiration = None or 10 # fixme
assert secret or originating_transfer
self.originating_transfer = originating_transfer # no sender == self initiated transfer
self.secret = secret
super(TransferTask, self).__init__()
print "INIT", self
self.transfermanager.on_task_started(self)
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__, pex(self.raiden.address))
@property
def isinitiator(self):
"whether this node initiated the transfer"
return not self.originating_transfer
def _run(self):
if self.isinitiator:
initiator = self.raiden.address
else:
initiator = self.originating_transfer.initiator
# look for shortest path
for path in self.assetmanager.channelgraph.get_paths(self.raiden.address, self.target):
print "TRYING {} with path {}".format(self, lpex(path))
assert path[0] == self.raiden.address
assert path[1] in self.assetmanager.channels
assert path[-1] == self.target
recipient = path[1]
# check if channel is active
if not self.assetmanager.channel_isactive(recipient):
continue
channel = self.assetmanager.channels[recipient]
# check if we have enough funds ,fixme add limit per transfer
if self.amount > channel.distributable:
continue
# calculate fee, calc expiration
t = channel.create_lockedtransfer(self.amount, self.expiration, self.hashlock)
t = t.to_mediatedtransfer(self.target, initiator=initiator, fee=0) # fixme fee
self.raiden.sign(t)
channel.register_transfer(t)
# send mediated transfer
msg = self.send_transfer(recipient, t, path)
print "SEND RETURNED {} {}".format(self, msg)
if isinstance(msg, CancelTransfer):
continue # try with next path
elif isinstance(msg, TransferTimeout):
# stale hashlock
if not self.isinitiator:
self.raiden.send(self.originating_transfer.sender, msg)
return self.on_completion(False)
elif isinstance(msg, Secret):
assert self.originating_transfer
assert msg.hashlock == self.hashlock
if self.originating_transfer.sender != self.originating_transfer.initiator:
fwd = Secret(msg.secret)
self.raiden.sign(fwd)
self.raiden.send(self.originating_transfer.sender, fwd)
else:
print "NOT FORWARDING SECRET TO ININTIATOR"
return self.on_completion(True)
elif isinstance(msg, SecretRequest):
assert self.isinitiator
assert msg.sender == self.target
msg = Secret(self.secret)
self.raiden.sign(msg)
self.raiden.send(self.target, msg)
# apply secret to own channel
channel.claim_locked(self.secret)
return self.on_completion(True)
# we did not find a path, send CancelTransfer
if self.originating_transfer:
channel = self.assetmanager.channels[self.originating_transfer.sender]
t = channel.create_canceltransfer(self.originating_transfer)
channel.register_transfer(t)
self.raiden.sign(t)
self.raiden.send(self.originating_transfer.sender, t)
return self.on_completion(False)
def on_event(self, msg):
print "SET EVENT {} {} {}".format(self, id(self.event), msg)
if self.event.ready():
print "ALREADY HAD EVENT {} {} now {}".format(self, self.event.get(), msg)
assert self.event and not self.event.ready()
self.event.set(msg)
def send_transfer(self, recipient, transfer, path):
self.event = AsyncResult() # http://www.gevent.org/gevent.event.html
self.raiden.send(recipient, transfer)
timeout = self.timeout_per_hop * (len(path) - 1) # fixme, consider no found paths
msg = self.event.wait(timeout)
print "HAVE EVENT {} {}".format(self, msg)
if msg is None: # timeout
print "TIMEOUT! " * 5
msg = TransferTimeout(echo=transfer.hash, hashlock=transfer.lock.hashlock)
self.raiden.sign(msg)
return msg
if isinstance(msg, CancelTransfer):
assert msg.hashlock == transfer.lock.hashlock
assert msg.amount == transfer.lock.amount
assert msg.recipient == transfer.sender == self.raiden.address
channel = self.assetmanager.channels[msg.recipient]
channel.register_transfer(msg)
return msg
# try with next path
elif isinstance(msg, TransferTimeout):
assert msg.echo == transfer.hash
return msg
# send back StaleHashLock, we need new hashlock
elif isinstance(msg, Secret):
# done exit
assert msg.hashlock == self.hashlock
# channel = self.assetmanager.channels[msg.recipient]
# channel.claim_locked(msg.secret) # fixme this is also done by assetmanager
return msg
elif isinstance(msg, SecretRequest):
# reveal secret
print "SECRETREQUEST RECEIVED {}".format(msg)
assert msg.sender == self.target
return msg
assert False, "Not Implemented"
class StartMediatedTransferTask(Task):
def __init__(self, transfermanager, amount, target):
super(StartMediatedTransferTask, self).__init__()
self.amount = amount
self.address = transfermanager.assetmanager.raiden.address
self.target = target
self.transfermanager = transfermanager
def __repr__(self):
return '<{} {}>'.format(
self.__class__.__name__,
pex(self.address),
)
def _run(self): # pylint: disable=method-hidden,too-many-locals
amount = self.amount
target = self.target
raiden = self.transfermanager.assetmanager.raiden
fee = 0
# there are no guarantees that the next_hop will follow the same route
routes = self.transfermanager.assetmanager.get_best_routes(
amount,
target,
lock_timeout=None,
)
for path, channel in routes:
# try a new secret
secret = sha3(hex(random.getrandbits(256)))
hashlock = sha3(secret)
self.transfermanager.register_task_for_hashlock(self, hashlock)
lock_expiration = (
raiden.chain.block_number() +
channel.settle_timeout -
raiden.config['reveal_timeout']
)
mediated_transfer = channel.create_mediatedtransfer(
raiden.address,
target,
fee,
amount,
lock_expiration,
hashlock,
)
raiden.sign(mediated_transfer)
channel.register_transfer(mediated_transfer)
response = self.send_and_wait_valid(raiden, path, mediated_transfer)
# `next_hop` timedout
if response is None:
self.transfermanager.on_hashlock_result(hashlock, False)
# someone down the line timedout / couldn't proceed
elif isinstance(response, (RefundTransfer, TransferTimeout)):
self.transfermanager.on_hashlock_result(hashlock, False)
# `target` received the MediatedTransfer
elif response.sender == target and isinstance(response, SecretRequest):
secret_message = Secret(secret)
raiden.sign(secret_message)
raiden.send(target, secret_message)
# this might cause synchronization problems, since it can take
# a while for the partner to receive the secret
channel.claim_locked(secret)
self.transfermanager.on_hashlock_result(hashlock, True)
# done, don't try a new path
return
else:
log.error('Unexpected response {}'.format(repr(response)))
self.transfermanager.on_hashlock_result(hashlock, False)
def send_and_wait_valid(self, raiden, path, mediated_transfer): # pylint: disable=no-self-use
""" Send the `mediated_transfer` and wait for either a message from
`target` or the `next_hop`.
Validate the message received and discards the invalid ones. The most
important case being next_hop sending a SecretRequest.
"""
message_timeout = raiden.config['msg_timeout']
next_hop = path[1]
target = path[-1]
transfer_details = 'path:{} hash:{}'.format(
lpex(path),
pex(mediated_transfer.hash),
)
log.debug('MEDIATED TRANSFER STARTED {}'.format(transfer_details))
current_time = time.time()
limit_time = current_time + message_timeout
self.event = AsyncResult()
raiden.send(next_hop, mediated_transfer)
while current_time <= limit_time:
response = self.event.wait(limit_time - current_time)
# critical write section
self.event = AsyncResult() # reset so that a new value can be received
# /critical write section
current_time = time.time()
if response is None:
log.debug('MEDIATED TRANSFER TIMED OUT {}'.format(transfer_details))
return None
if response.sender == next_hop:
if isinstance(response, (RefundTransfer, TransferTimeout)):
return response
else:
log.info('Partner {} sent an invalid message'.format(pex(next_hop)))
return None
if response.sender == target:
if isinstance(response, SecretRequest):
return response
else:
log.info('target {} sent an invalid message'.format(pex(target)))
return None
log.error('Invalid message ignoring. {}'.format(repr(response)))
return None
class AlarmTask(Task):
""" Task to notify when a block is mined. """
def __init__(self, chain):
super(AlarmTask, self).__init__()
self.callbacks = list()
self.stop_event = AsyncResult()
self.wait_time = 0.5
self.chain = chain
self.last_block_number = self.chain.block_number()
def register_callback(self, callback):
""" Register a new callback.
Note:
This callback will be executed in the AlarmTask context and for
this reason it should not block, otherwise we can miss block
changes.
"""
if not callable(callback):
raise ValueError('callback is not a callable')
self.callbacks.append(callback)
def _run(self): # pylint: disable=method-hidden
stop = None
result = None
last_loop = time.time()
log.debug('starting block number', block_number=self.last_block_number)
while stop is None:
current_block = self.chain.block_number()
if current_block > self.last_block_number + 1:
difference = current_block - self.last_block_number - 1
log.error(
'alarm missed %s blocks',
difference,
)
if current_block != self.last_block_number:
self.last_block_number = current_block
log.debug('new block',
number=current_block,
timestamp=last_loop)
remove = list()
for callback in self.callbacks:
try:
result = callback(current_block)
except:
log.exception('unexpected exception on alarm')
else:
if result is REMOVE_CALLBACK:
remove.append(callback)
for callback in remove:
self.callbacks.remove(callback)
# we want this task to iterate in the tick of `wait_time`, so take
# into account how long we spent executing one tick.
work_time = time.time() - last_loop
if work_time > self.wait_time:
log.warning(
'alarm loop is taking longer than the wait time',
work_time=work_time,
wait_time=self.wait_time,
)
sleep_time = 0.001
else:
sleep_time = self.wait_time - work_time
stop = self.stop_event.wait(sleep_time)
last_loop = time.time()
def stop(self):
self.stop_event.set(True)
class AlarmTask(gevent.Greenlet):
""" Task to notify when a block is mined. """
def __init__(self, chain):
super().__init__()
# TODO: Start with a larger sleep_time and decrease it as the
# probability of a new block increases.
sleep_time = 0.5
self.callbacks = list()
self.chain = chain
self.chain_id = None
self.last_block_number = None
self.stop_event = AsyncResult()
self.sleep_time = sleep_time
def _run(self): # pylint: disable=method-hidden
try:
self.loop_until_stop()
except RaidenShuttingDown:
pass
finally:
self.callbacks = list()
def register_callback(self, callback):
""" Register a new callback.
Note:
The callback will be executed in the AlarmTask context and for
this reason it should not block, otherwise we can miss block
changes.
"""
if not callable(callback):
raise ValueError('callback is not a callable')
self.callbacks.append(callback)
def remove_callback(self, callback):
"""Remove callback from the list of callbacks if it exists"""
if callback in self.callbacks:
self.callbacks.remove(callback)
def loop_until_stop(self):
# The AlarmTask must have completed its first_run() before starting
# the background greenlet.
#
# This is required because the first run will synchronize the node with
# the blockchain since the last run.
assert self.chain_id, 'chain_id not set'
assert self.last_block_number, 'last_block_number not set'
chain_id = self.chain_id
sleep_time = self.sleep_time
while self.stop_event.wait(sleep_time) is not True:
last_block_number = self.last_block_number
current_block = self.chain.block_number()
if chain_id != self.chain.network_id:
raise RuntimeError(
'Changing the underlying blockchain while the Raiden node is running '
'is not supported.',
)
if current_block != last_block_number:
log.debug('new block', number=current_block)
if current_block > last_block_number + 1:
missed_blocks = current_block - last_block_number - 1
log.error(
'missed blocks',
missed_blocks=missed_blocks,
current_block=current_block,
)
self.run_callbacks(current_block, chain_id)
def first_run(self):
# callbacks must be executed during the first run to update the node state
assert self.callbacks, 'callbacks not set'
chain_id = self.chain.network_id
current_block = self.chain.block_number()
log.debug('starting at block number', current_block=current_block)
self.run_callbacks(current_block, chain_id)
self.chain_id = chain_id
def run_callbacks(self, current_block, chain_id):
remove = list()
for callback in self.callbacks:
result = callback(current_block, chain_id)
if result is REMOVE_CALLBACK:
remove.append(callback)
for callback in remove:
self.callbacks.remove(callback)
self.last_block_number = current_block
def stop_async(self):
self.stop_event.set(True)
class MediateTransferTask(Task): # pylint: disable=too-many-instance-attributes
def __init__(self, transfermanager, originating_transfer, fee):
super(MediateTransferTask, self).__init__()
self.address = transfermanager.assetmanager.raiden.address
self.transfermanager = transfermanager
self.fee = fee
self.originating_transfer = originating_transfer
hashlock = originating_transfer.lock.hashlock
self.transfermanager.register_task_for_hashlock(self, hashlock)
def __repr__(self):
return '<{} {}>'.format(self.__class__.__name__, pex(self.address))
def _run(self): # pylint: disable=method-hidden,too-many-locals,too-many-branches,too-many-statements
fee = self.fee
transfer = self.originating_transfer
assetmanager = self.transfermanager.assetmanager
raiden = assetmanager.raiden
originating_channel = assetmanager.partneraddress_channel[
transfer.sender]
assetmanager.register_channel_for_hashlock(
originating_channel,
transfer.lock.hashlock,
)
lock_expiration = transfer.lock.expiration - raiden.config[
'reveal_timeout']
lock_timeout = lock_expiration - raiden.chain.block_number()
# there are no guarantees that the next_hop will follow the same route
routes = assetmanager.get_best_routes(
transfer.lock.amount,
transfer.target,
lock_timeout,
)
if log.isEnabledFor(logging.DEBUG):
log.debug(
'MEDIATED TRANSFER initiator:%s node:%s target:%s',
pex(transfer.initiator),
pex(self.address),
pex(transfer.target),
)
for path, forward_channel in routes:
next_hop = path[1]
mediated_transfer = forward_channel.create_mediatedtransfer(
transfer.initiator,
transfer.target,
fee,
transfer.lock.amount,
lock_expiration,
transfer.lock.hashlock,
)
raiden.sign(mediated_transfer)
if log.isEnabledFor(logging.DEBUG):
log.debug(
'MEDIATED TRANSFER NEW PATH path:%s hashlock:%s',
lpex(path),
pex(transfer.lock.hashlock),
)
# Using assetmanager to register the interest because it outlives
# this task, the secret handling will happend only _once_
assetmanager.register_channel_for_hashlock(
forward_channel,
transfer.lock.hashlock,
)
forward_channel.register_transfer(mediated_transfer)
response = self.send_and_wait_valid(raiden, path,
mediated_transfer)
if response is None:
timeout_message = forward_channel.create_timeouttransfer_for(
transfer)
raiden.send_async(transfer.sender, timeout_message)
self.transfermanager.on_hashlock_result(
transfer.lock.hashlock, False)
return
if isinstance(response, RefundTransfer):
if response.lock.amount != transfer.amount:
log.info(
'Partner %s sent an refund message with an invalid amount',
pex(next_hop),
)
timeout_message = forward_channel.create_timeouttransfer_for(
transfer)
raiden.send_async(transfer.sender, timeout_message)
self.transfermanager.on_hashlock_result(
transfer.lock.hashlock, False)
return
else:
forward_channel.register_transfer(response)
elif isinstance(response, Secret):
# update all channels and propagate the secret (this doesnt claim the lock yet)
assetmanager.handle_secret(response.secret)
# wait for the secret from `sender`
while True:
response = self.response_message.wait()
# critical write section
self.response_message = AsyncResult()
# /critical write section
# NOTE: this relies on the fact RaindenService dispatches
# messages based on the `hashlock` calculated from the
# secret, so we know this `response` message secret matches
# the secret from the `next_hop`
if isinstance(
response,
Secret) and response.sender == transfer.sender:
originating_channel.claim_lock(response.secret)
self.transfermanager.on_hashlock_result(
transfer.lock.hashlock, True)
return
# No suitable path avaiable (e.g. insufficient distributable, no active node)
# Send RefundTransfer to the originating node, this has the effect of
# backtracking in the graph search of the raiden network.
from_address = transfer.sender
from_channel = assetmanager.partneraddress_channel[from_address]
refund_transfer = from_channel.create_refundtransfer_for(transfer)
from_channel.register_transfer(refund_transfer)
raiden.sign(refund_transfer)
raiden.send_async(from_address, refund_transfer)
log.debug(
'REFUND MEDIATED TRANSFER from=%s node:%s hashlock:%s',
pex(from_address),
pex(raiden.address),
pex(transfer.lock.hashlock),
)
self.transfermanager.on_hashlock_result(transfer.lock.hashlock, False)
return
def send_and_wait_valid(self, raiden, path, mediated_transfer):
message_timeout = raiden.config['msg_timeout']
next_hop = path[1]
current_time = time.time()
limit_time = current_time + message_timeout
self.response_message = AsyncResult()
raiden.send_async(next_hop, mediated_transfer)
while current_time <= limit_time:
response = self.response_message.wait(limit_time - current_time)
# critical write section
self.response_message = AsyncResult(
) # reset so that a new value can be received
# /critical write section
current_time = time.time()
if response is None:
log.error(
'MEDIATED TRANSFER TIMED OUT node:%s timeout:%s msghash:%s hashlock:%s',
pex(raiden.address),
message_timeout,
pex(mediated_transfer.hash),
pex(mediated_transfer.lock.hashlock),
)
return None
if isinstance(response, Secret):
if sha3(response.secret) != mediated_transfer.lock.hashlock:
log.error('Secret doesnt match the hashlock, ignoring.')
continue
return response
if response.target != raiden.address or response.sender != next_hop:
log.error('Invalid message supplied to the task. %s',
repr(response))
continue
if isinstance(response, RefundTransfer):
return response
log.error('Partner sent an invalid message. %s', repr(response))
return None
class MediateTransferTask(Task): # pylint: disable=too-many-instance-attributes
def __init__(self, transfermanager, originating_transfer, fee):
super(MediateTransferTask, self).__init__()
self.address = transfermanager.assetmanager.raiden.address
self.transfermanager = transfermanager
self.fee = fee
self.originating_transfer = originating_transfer
hashlock = originating_transfer.lock.hashlock
self.transfermanager.register_task_for_hashlock(self, hashlock)
def __repr__(self):
return '<{} {}>'.format(
self.__class__.__name__,
pex(self.address)
)
def _run(self): # pylint: disable=method-hidden,too-many-locals,too-many-branches,too-many-statements
fee = self.fee
transfer = self.originating_transfer
assetmanager = self.transfermanager.assetmanager
raiden = assetmanager.raiden
originating_channel = assetmanager.partneraddress_channel[transfer.sender]
assetmanager.register_channel_for_hashlock(
originating_channel,
transfer.lock.hashlock,
)
lock_expiration = transfer.lock.expiration - raiden.config['reveal_timeout']
lock_timeout = lock_expiration - raiden.chain.block_number()
# there are no guarantees that the next_hop will follow the same route
routes = assetmanager.get_best_routes(
transfer.lock.amount,
transfer.target,
lock_timeout,
)
for path, channel in routes:
next_hop = path[1]
mediated_transfer = channel.create_mediatedtransfer(
transfer.initiator,
transfer.target,
fee,
transfer.lock.amount,
lock_expiration,
transfer.lock.hashlock,
)
raiden.sign(mediated_transfer)
assetmanager.register_channel_for_hashlock(
channel,
transfer.lock.hashlock,
)
channel.register_transfer(mediated_transfer)
response = self.send_and_wait_valid(raiden, path, mediated_transfer)
if response is None:
timeout = channel.create_timeouttransfer_for(transfer)
raiden.send(transfer.sender, timeout)
self.transfermanager.on_hashlock_result(transfer.hashlock, False)
return
if isinstance(response, RefundTransfer):
if response.lock.amount != transfer.amount:
log.info('Partner {} sent an refund message with an invalid amount'.format(
pex(next_hop),
))
timeout = channel.create_timeouttransfer_for(transfer)
raiden.send(transfer.sender, timeout)
self.transfermanager.on_hashlock_result(transfer.hashlock, False)
return
else:
channel.register_transfer(response)
elif isinstance(response, Secret):
# update all channels and propagate the secret
assetmanager.register_secret(response.secret)
self.transfermanager.on_hashlock_result(transfer.lock.hashlock, True)
return
# No suitable path avaiable (e.g. insufficient distributable, no active node)
# Send RefundTransfer to the originating node, this has the effect of
# backtracking in the graph search of the raiden network.
from_address = transfer.sender
from_channel = assetmanager.partneraddress_channel[from_address]
refund_transfer = from_channel.create_refundtransfer_for(transfer)
from_channel.register_transfer(refund_transfer)
raiden.sign(refund_transfer)
raiden.send(from_address, refund_transfer)
log.debug('REFUND MEDIATED TRANSFER from={} {}'.format(
pex(from_address),
pex(raiden.address),
))
self.transfermanager.on_hashlock_result(transfer.hashlock, False)
return
def send_and_wait_valid(self, raiden, path, mediated_transfer):
message_timeout = raiden.config['msg_timeout']
next_hop = path[1]
transfer_details = 'path:{} hash:{} initiator:{}'.format(
lpex(path),
pex(mediated_transfer.hash),
pex(mediated_transfer.initiator),
)
log.debug('MEDIATED TRANSFER {}'.format(transfer_details))
current_time = time.time()
limit_time = current_time + message_timeout
self.event = AsyncResult()
raiden.send(next_hop, mediated_transfer)
while current_time <= limit_time:
response = self.event.wait(limit_time - current_time)
# critical write section
self.event = AsyncResult() # reset so that a new value can be received
# /critical write section
current_time = time.time()
if response is None:
log.error('MEDIATED TRANSFER TIMED OUT {} timeout:{}'.format(
transfer_details,
message_timeout,
))
return None
if isinstance(response, Secret):
if response.hashlock != mediated_transfer.lock.hashlock:
log.error('Secret doesnt match the hashlock, ignoring.')
continue
return response
if response.target != raiden.address or response.sender != next_hop:
log.error('Invalid message supplied to the task. {}'.format(repr(response)))
continue
if isinstance(response, RefundTransfer):
return response
log.error('Partner sent an invalid message. {}'.format(repr(response)))
return None
class LogListenerTask(Task):
""" Task for polling for filter changes. """
def __init__(self,
listener_name,
filter_,
callback,
contract_translator,
events_poll_timeout=DEFAULT_EVENTS_POLL_TIMEOUT):
"""
Args:
listener_name (str): A name to distinguish listener tasks.
filter_ (raiden.network.rpc.client.Filter): A proxy for calling the
blockchain's filter api.
callback (function): A function to be called once an event happens.
contract_translator (ethereum.abi.ContractTranslator): A contract
translator to decode the event data.
events_poll_timeout (float): How long the tasks should sleep before
polling again.
"""
super(LogListenerTask, self).__init__()
self.listener_name = listener_name
self.filter_ = filter_
self.callback = callback
self.contract_translator = contract_translator
self.stop_event = AsyncResult()
self.sleep_time = events_poll_timeout
# exposes the AsyncResult timer, this allows us to raise the timeout
# inside this Task to force an update:
#
# task.kill(task.timeout)
#
self.timeout = None
def __repr__(self):
return '<LogListenerTask {}>'.format(self.listener_name)
def _run(self): # pylint: disable=method-hidden
stop = None
while stop is None:
filter_changes = self.filter_.changes()
for log_event in filter_changes:
log.debug('New Events', task=self.listener_name)
event = self.contract_translator.decode_event(
log_event['topics'],
log_event['data'],
)
if event is not None:
originating_contract = log_event['address']
try:
self.callback(originating_contract, event)
except:
log.exception('unexpected exception on log listener')
self.timeout = Timeout(
self.sleep_time) # wait() will call cancel()
stop = self.stop_event.wait(self.timeout)
def stop(self):
self.stop_event.set(True)
class ClientService(object):
def __init__(self, _websocket, user_id, _config):
self._user_id = user_id
self._config = _config
self._websocket = _websocket
assert isinstance(self._websocket, WebSocket)
matches = _config.get(ConfigField.MATCH) # 订阅的条件， 目前是regex的字符串列表
fields = _config.get(ConfigField.FIELD) # 返回的订阅数据字段列表
self.compress_type = CompressValues.DEFAULT \
if ConfigField.COMPRESS in _config and _config[ConfigField.COMPRESS] else None
self._patterns = []
for item in matches:
if isinstance(item, basestring):
self._patterns.append(re.compile(item))
if item not in self.all_patterns:
self.all_patterns[item] = {self._user_id}
else:
self.all_patterns[item].add(self._user_id)
self._fields = set([item for item in fields if item in ALL_FIELDS])
# 存在file_id表明数据被持久化储存
self._fields.add(FILE_ID)
self._publish = AsyncResult()
self.clients[user_id] = self
def get_publish(self):
result = self._publish.wait()
self._publish = AsyncResult()
return result
fields = property(lambda o: o._fields)
publish = property(get_publish, lambda o, v: o._publish.set(v))
clients = {}
all_patterns = {}
def close(self):
for k, v in self.all_patterns.items():
assert isinstance(v, set)
if self._user_id in v:
v.remove(self._user_id)
if len(v) == 0:
self.all_patterns.pop(k)
self.clients.pop(self._user_id, None)
def get(self):
# block
return self._publish.get()
def match(self, url):
for p in self._patterns:
if p.match(url):
return True
return False
@classmethod
def add(cls, **kwargs):
# 找到所有需要广播的用户
url = kwargs.get("url")
all_user_id_set = set()
for pattern_str, user_id_set in cls.all_patterns.iteritems():
assert isinstance(user_id_set, set)
# 使用系统自带的正则缓存
p_obj = re.compile(pattern_str)
if p_obj.match(url):
all_user_id_set.update(user_id_set)
# 数据格式化
if FILE_ID in kwargs:
kwargs[FILE_ID] = str(kwargs[FILE_ID])
for name in kwargs.keys():
if name in BASE64_FIELDS:
# 编码成base64 方便转换成json
kwargs[name] = kwargs[name].encode("base64")
elif name in DICT_FIELDS:
# 将请求头和返回头转换成dict
kwargs[name] = dict(kwargs[name])
# 广播
for user_id in all_user_id_set:
obj = cls.clients[user_id]
obj.publish = {
k: v
for k, v in kwargs.iteritems() if k in obj.fields
}
def one_send(self):
# 阻塞
result = self.publish
again_send = {}
for k, v in result.items():
if k in BIG_DATA_FIELDS:
again_send[k] = result.pop(k)
again_data = ""
if len(again_send) > 0:
again_data = json.dumps(again_send)
if self.compress_type == CompressValues.DEFAULT:
again_data = zip_compress(again_data)
result[AGAIN_LENGTH] = len(again_data)
self._websocket.send(json.dumps(result))
if len(again_send) > 0:
sio = StringIO(again_data)
send_time = len(again_data) / (ONE_SEND_LENGTH * 1.0)
for _ in xrange(
int(send_time) +
1 if send_time > int(send_time) else int(send_time)):
self._websocket.send(sio.read(ONE_SEND_LENGTH), True)
class AlarmTask(Task):
""" Task to notify when a block is mined. """
def __init__(self, chain):
super(AlarmTask, self).__init__()
self.callbacks = list()
self.stop_event = AsyncResult()
self.chain = chain
self.last_block_number = None
# TODO: Start with a larger wait_time and decrease it as the
# probability of a new block increases.
self.wait_time = 0.5
self.last_loop = time.time()
def register_callback(self, callback):
""" Register a new callback.
Note:
The callback will be executed in the AlarmTask context and for
this reason it should not block, otherwise we can miss block
changes.
"""
if not callable(callback):
raise ValueError('callback is not a callable')
self.callbacks.append(callback)
def remove_callback(self, callback):
"""Remove callback from the list of callbacks if it exists"""
if callback in self.callbacks:
self.callbacks.remove(callback)
def _run(self): # pylint: disable=method-hidden
log.debug('starting block number', block_number=self.last_block_number)
sleep_time = 0
while self.stop_event.wait(sleep_time) is not True:
self.poll_for_new_block()
# we want this task to iterate in the tick of `wait_time`, so take
# into account how long we spent executing one tick.
self.last_loop = time.time()
work_time = self.last_loop - self.last_loop
if work_time > self.wait_time:
log.warning(
'alarm loop is taking longer than the wait time',
work_time=work_time,
wait_time=self.wait_time,
)
sleep_time = 0.001
else:
sleep_time = self.wait_time - work_time
# stopping
self.callbacks = list()
def poll_for_new_block(self):
current_block = self.chain.block_number()
if current_block > self.last_block_number + 1:
difference = current_block - self.last_block_number - 1
log.error(
'alarm missed %s blocks',
difference,
)
if current_block != self.last_block_number:
log.debug(
'new block',
number=current_block,
timestamp=self.last_loop,
)
self.last_block_number = current_block
remove = list()
for callback in self.callbacks:
try:
result = callback(current_block)
except: # pylint: disable=bare-except
log.exception('unexpected exception on alarm')
else:
if result is REMOVE_CALLBACK:
remove.append(callback)
for callback in remove:
self.callbacks.remove(callback)
def start(self):
self.last_block_number = self.chain.block_number()
super(AlarmTask, self).start()
def stop_and_wait(self):
self.stop_event.set(True)
gevent.wait(self)
def stop_async(self):
self.stop_event.set(True)
class DolphinConnection(object):
def __init__(self, host="localhost", port=6000):
'''
Creating a new DolphinConnection instance,
pointing to the DolphinConnection Server specified by host and port.
The connection must be established explicitly with connect().
host and port can be overwritten, followed by another connect()
call to reconnect.
'''
self.host = host
self.port = port
self._connected = False
self._sock = None
self._cFunc = None
self._dcFunc = None
self._callbacks = {}
self._buf = ""
self._sep = "\n"
self._feedback = AsyncResult()
self._feedback.set(None)
def isConnected(self):
'''
Returns whether the DolphinConnection instance is connected to the
corresponding server defined by host and port.
'''
return self._connected
def connect(self):
'''
Tries to establish a connection to the server.
If it succeeds, the onConnect callback will be called.
If it fails, the onDisconnect callback will be called.
'''
self.disconnect()
self._connected = True
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._sock.connect((self.host, self.port))
logger.info("DolphinConnection connection to %s:%d established! " +
"Ready for work!", self.host, self.port)
gevent.spawn(self._recv)
if self._cFunc:
self._cFunc(self)
except socket.error:
logger.info("DolphinConnection connection to %s:%d failed.",
self.host, self.port)
self._disconnect(DisconnectReason.CONNECTION_FAILED)
def disconnect(self):
'''
Disconnects the socket from the server.
The onDisconnect callback will be called with CONNECTION_CLOSED_BY_HOST
'''
self._disconnect(DisconnectReason.CONNECTION_CLOSED_BY_HOST)
def _disconnect(self, reason):
if not self._connected:
return
self._connected = False
self._feedback.set(False)
try:
self._sock.close()
except:
pass
if self._dcFunc:
self._dcFunc(self, reason)
def onConnect(self, func):
'''
Sets the callback that will be called after a connection
has been successfully established.
Callback is initially None, and can again be assigned to None.
'''
if not hasattr(func, '__call__'):
raise ValueError("onDisconnect callback must be callable.")
self._cFunc = func
def onDisconnect(self, func):
'''
Sets the callback that will be called after a connection attempt fails,
an active connection gets closed or the connection gets lost.
A DisconnectReason enum will be the parameter.
Callback is initially None, and can again be assigned to None.
'''
if not hasattr(func, '__call__'):
raise ValueError("onDisconnect callback must be callable.")
self._dcFunc = func
def startBatch(self):
'''
Call this function to send following commands in a batch.
All following commands are guaranteed to be executed at once in
Dolphin. Is done by buffering and not executing anything until
endBatch() is called.
'''
self._sep = ";"
def endBatch(self):
'''
Ends the batch started with startBatch().
All buffered commands gets executed now and no more buffering is done.
'''
self._sep = "\n"
self._cmd("")
def volume(self, v):
'''
Sets Dolphin's Audio.
:param v: 0-100, audio level
'''
self._cmd("VOLUME %d" % v)
def write(self, mode, addr, val):
'''
Sends a command to write <mode> bytes of data to the given address.
<mode> must be 8, 16 or 32.
'''
self._cmd("WRITE %d %d %d" % (mode, addr, val))
def writeMulti(self, addr, vals):
'''
Sends a command to write the bytes <vals>, starting at address <addr>.
'''
self._cmd("WRITE_MULTI %d %s" % (addr, " ".join(str(v) for v in vals)))
def read(self, mode, addr, callback):
'''
Sends a command to send back <mode> bytes of data at the given address.
The given callback function gets called with the returned value as
parameter.
<mode> must be 8, 16 or 32.
'''
self._reg_callback(addr, callback, False)
self._cmd("READ %d %d" % (mode, addr))
def _subscribe(self, mode, addr, callback):
'''
Sends a command to send back <mode> bytes of data at the given address,
repeating each time the value changes.
The given callback function gets called with the returned value as
parameter.
<mode> must be 8, 16 or 32.
'''
self._reg_callback(addr, callback, True)
self._cmd("SUBSCRIBE %d %d" % (mode, addr))
def _subscribeMulti(self, size, addr, callback):
'''
Sends a command to send back <size> bytes of data starting at the
given address,
repeating each time the value changes. Useful for strings and arrays.
The given callback function gets called with the returned values in a
list as parameter.
'''
self._reg_callback(addr, callback, True)
self._cmd("SUBSCRIBE_MULTI %d %d" % (size, addr))
def write8(self, addr, val):
'''
Sends a command to write 8 bytes of data to the given address.
'''
self.write(8, addr, val)
def write16(self, addr, val):
'''
Sends a command to write 16 bytes of data to the given address.
'''
self.write(16, addr, val)
def write32(self, addr, val):
'''
Sends a command to write 32 bytes of data to the given address.
'''
self.write(32, addr, val)
def read8(self, addr, callback):
'''
Sends a command to send back 8 bytes of data at the given address.
The given callback function gets called with the returned value as
parameter.
'''
self.read(8, addr, callback)
def read16(self, addr, callback):
'''
Sends a command to send back 16 bytes of data at the given address.
The given callback function gets called with the returned value as
parameter.
'''
self.read(16, addr, callback)
def read32(self, addr, callback):
'''
Sends a command to send back 32 bytes of data at the given address.
The given callback function gets called with the returned value as
parameter.
'''
if addr % 4 != 0:
raise ValueError("Read32 address must be whole word; " +
"multiple of 4")
self.read(32, addr, callback)
def subscribe8(self, addr, callback):
'''
Sends a command to send back 8 bytes of data at the given address,
repeating each time the value changes.
The given callback function gets called with the returned value as
parameter.
'''
self._subscribe(8, addr, callback)
def subscribe16(self, addr, callback):
'''
Sends a command to send back 16 bytes of data at the given address,
repeating each time the value changes.
The given callback function gets called with the returned value as
parameter.
'''
self._subscribe(16, addr, callback)
def subscribe32(self, addr, callback):
'''
Sends a command to send back 32 bytes of data at the given address,
repeating each time the value changes.
The given callback function gets called with the returned value as
parameter.
'''
if addr % 4 != 0:
raise ValueError("Read address must be whole word; " +
"multiple of 4")
self._subscribe(32, addr, callback)
def subscribeMulti(self, size, addr, callback):
'''
Sends a command to send back <size> bytes of data starting at the
given address,
repeating each time any value changes. Useful for strings or arrays.
The given callback function gets called with the returned values in a
list as parameter.
'''
self._subscribeMulti(size, addr, callback)
def wiiButton(self, wiimoteIndex, buttonstates):
'''
Sends 16 bit of data representing some buttonstates of the Wiimote.
NOTE: The real or emulated wiimote dolphin uses gets hijacked for only
roughly half a second.
After this time that wiimote handled by dolphin starts to send
it's buttonstates again.
:param wiimoteIndex: 0-3, index of the wiimote to emulate.
:param buttonstates: bitmask of the buttonstates,
see http://wiibrew.org/wiki/Wiimote#Buttons for more info
'''
self._cmd("BUTTONSTATES_WII %d %d" % (wiimoteIndex, buttonstates))
def gcButton(self, gcpadIndex, buttonstates, stickX=0.0, stickY=0.0,
substickX=0.0, substickY=0.0):
'''
Sends 16 bit of data and 2 floats representing some buttonstates of
the GCPad.
NOTE: The real or emulated gcpad dolphin uses gets hijacked for only
roughly half a second.
After this time that gcpad handled by dolphin starts to send
it's buttonstates again.
:param gcpadIndex: 0-3, index of the gcpad to emulate.
:param buttonstates: bitmask of the buttonstates,
see http://pastebin.com/raw.php?i=4txWae07 for more info
:param stickX: between -1.0 and 1.0, x-position of the main stick,
0 is neutral
:param stickY: between -1.0 and 1.0, y-position of the main stick,
0 is neutral
:param substickX: between -1.0 and 1.0, x-position of the c-stick,
0 is neutral
:param substickY: between -1.0 and 1.0, y-position of the c-stick,
0 is neutral
'''
self._cmd("BUTTONSTATES_GC %d %d %f %f %f %f" % (gcpadIndex,
buttonstates,
stickX,
stickY,
substickX,
substickY))
def pause(self):
'''
Tells Dolphin to pause the current emulation.
Resume with resume()
'''
self._cmd("PAUSE")
def resume(self):
'''
Tells Dolphin to resume the current emulation.
'''
self._cmd("RESUME")
def reset(self):
'''
Tells Dolphin to push the reset button.
'''
self._cmd("RESET")
def save(self, filename):
'''
Tells Dolphin to make a savestate and save it to <filename>.
'''
if any(c in filename for c in "?\"<>|"):
raise ValueError("filename must not contain any of the " +
"following: :?\"<> | ")
self._cmd("SAVE %s" % filename)
def load(self, filename):
'''
Tells Dolphin to load the savestate located at <filename>.
This function will block until feedback as arrived
and will then return true if it succeded, else false.
CAUTION: Will permanently block if dolphin was paused :(
'''
if any(c in filename for c in "?\"<>|"):
raise ValueError("filename must not contain any of the " +
"following: ?\"<> | ")
return self._cmd("LOAD %s" % filename, True)
def stop(self):
'''
Stops the current emulation. DolphinWatch does NOT support starting a
new game then.
To change the game, use insert() to insert a new iso and then reset().
'''
self._cmd("STOP")
def insert(self, filename):
'''
Inserts up a new game (iso).
:param filename: The file (iso e.g.) to be loaded. Relative do dolphin.
CAUTION: Running games can crash if the iso changes while running.
To change a game, pause, then insert, and after a bit reset the game.
'''
if any(c in filename for c in "?\"<>|"):
raise ValueError("filename must not contain any of the " +
"following: ?\"<> | ")
self._cmd("INSERT %s" % filename)
######################################
# private methods below
def _cmd(self, cmd, feedback=False):
if not self._connected:
raise socket.error("DolphinConnection is not connected and " +
"therefore cannot perform actions!")
if feedback:
try:
self._feedback.wait(1.0)
except gevent.Timeout:
pass
# TODO got locked up :(
self._feedback = AsyncResult()
self._sock.send((cmd + self._sep).encode())
r = self._feedback.get(True)
return r
else:
self._sock.send((cmd + self._sep).encode())
return True
def _reg_callback(self, addr, func, _subscribe=False):
self._callbacks[addr] = (func, _subscribe)
def _dereg_callback(self, addr):
self._callbacks.pop(addr)
def _process(self, line):
parts = line.split(" ")
if parts[0] == "MEM":
addr = int(parts[1])
val = int(parts[2])
callback = self._callbacks.get(addr)
if callback:
if not callback[1]:
self._dereg_callback(addr)
gevent.spawn(callback[0], val)
else:
logger.warning("No recipient for address 0x%x, value 0x%x",
addr, val)
elif parts[0] == "MEM_MULTI":
addr = int(parts[1])
data = [int(v) for v in parts[2:]]
callback = self._callbacks.get(addr)
if callback:
if not callback[1]:
self._dereg_callback(addr)
gevent.spawn(callback[0], data)
else:
logger.warning("No recipient for address 0x%x, data %s",
addr, data)
elif parts[0] == "FAIL":
self._feedback.set(False)
elif parts[0] == "SUCCESS":
self._feedback.set(True)
elif parts[0] == "LOG":
level = _log_translation[int(parts[1])]
dolphin_logger.log(level, " ".join(parts[2:]))
else:
logger.warning("Unknown incoming DolphinWatch command: %s", line)
def _recv(self):
while self._connected:
try:
data = self._sock.recv(1024)
if not data:
logger.info("DolphinConnection connection closed")
self._disconnect(DisconnectReason.CONNECTION_CLOSED_BY_PEER)
return
self._buf += data.decode()
except socket.error:
logger.warning("DolphinConnection connection lost")
self._disconnect(DisconnectReason.CONNECTION_LOST)
return
*lines, rest = self._buf.split("\n")
self._buf = rest
for line in lines:
self._process(line.strip())
