def polling_app(environ, start_response):
# If we're sitting behind an elb providing HTTPS:
forwarded_proto = environ.get('HTTP_X_FORWARDED_PROTO')
if forwarded_proto:
environ['wsgi.url_scheme'] = forwarded_proto
path = environ['PATH_INFO']
if path.endswith('/longpoll'):
client_gen = int(environ.get('HTTP_X_GENERATION', 0))
if client_gen:
zoid = int(environ['HTTP_X_GENERATION_ZOID'])
event = gevent.event.Event()
pollers[zoid].append(event)
with db.transaction() as conn:
gen = conn.get(p64(zoid)).generation
if gen <= client_gen:
# Wait for updates
event.wait(300)
else:
try:
pollers[zoid].remove(event)
except IndexError:
pass
return app(environ, start_response)
def yield_future(future, loop=None):
"""Wait for a future, a task, or a coroutine object from a greenlet.
Yield control other eligible greenlet until the future is done (finished
successfully or failed with an exception).
Return the result or raise the exception of the future.
The function must not be called from the greenlet running the aiogreen
event loop.
"""
if hasattr(asyncio, "ensure_future"):
ensure_future = asyncio.ensure_future
else: # use of async keyword has been Deprecated since Python 3.4.4
ensure_future = getattr(asyncio, "async")
future = ensure_future(future, loop=loop)
if future._loop._greenlet == gevent.getcurrent():
raise RuntimeError("yield_future() must not be called from "
"the greenlet of the aiogreen event loop")
event = gevent.event.Event()
def wakeup_event(fut):
event.set()
future.add_done_callback(wakeup_event)
event.wait()
return future.result()
def _get(self, client, timeout, count):
if not timeout:
timeout = 1
exp = datetime.utcnow() + timedelta(seconds=timeout)
messages = []
event = gevent.event.Event()
self.q.put((exp, count, client, messages, event))
event.wait()
return messages
def _get(self, client, timeout, count):
if not timeout:
timeout = 1
exp = datetime.utcnow() + timedelta(seconds=timeout)
messages = []
event = gevent.event.Event()
self.q.put((exp, count, client, messages, event))
event.wait()
return messages
def callwhen(event, fun):
while True:
try:
event.wait()
event.clear()
fun()
except gevent.GreenletExit:
raise
except Exception:
traceback.print_exc()
pass
def getblocktemplate(self, params, uri, auth):
"""For worker"""
longpollid = 'init'
mode = 'template' # For older client
data = None
for i in params:
if 'longpollid' in i:
longpollid = i['longpollid']
if 'mode' in i:
mode = i['mode']
if 'data' in i:
data = i['data']
if mode == 'submit':
try:
block = util.h2b(data)
sizelen, tx_size = util.decode_size(block[80:89])
ptr = 80 + sizelen
txlen = CoinbaseTransaction.verify(block[ptr:],
self.generation_pubkey)
result = self.process_block(block[:ptr + txlen], auth)
if result:
return None
except:
logger.error("Exception while processing block")
logger.error(traceback.format_exc())
return 'Rejected'
if longpollid != 'init' or uri == config.longpoll_uri:
event = gevent.event.Event()
self.add_longpoll_event(event)
event.wait()
longpollid = self.get_work_id()
coinbase_tx = self.create_coinbase_tx(b'', b'')
block_template = {
k: self.block_template[k]
for k in self.block_template
if k not in ['coinbasevalue', 'coinbaseaux', 'coinbaseflags']
}
block_template['target'] = util.b2h(
self._get_target(auth['difficulty']))
block_template['mutable'] = ["coinbase/append", "submit/coinbase"]
block_template['transactions'] = [x.serialize() for x in self.tx]
block_template['coinbasetxn'] = coinbase_tx.serialize()
#Long polling extension
block_template['longpollid'] = longpollid
block_template['expires'] = 120
block_template['submitold'] = True
block_template['longpolluri'] = config.longpoll_uri
return block_template
def getblocktemplate(self, params, uri, auth):
"""For worker"""
longpollid = 'init'
mode = 'template' # For older client
data = None
for i in params:
if 'longpollid' in i:
longpollid = i['longpollid']
if 'mode' in i:
mode = i['mode']
if 'data' in i:
data = i['data']
if mode == 'submit':
try:
block = util.h2b(data)
sizelen, tx_size = util.decode_size(block[80:89])
ptr = 80 + sizelen
txlen = CoinbaseTransaction.verify(block[ptr:],
self.generation_pubkey)
result = self.process_block(block[:ptr+txlen], auth)
if result:
return None
except:
logger.error("Exception while processing block")
logger.error(traceback.format_exc())
return 'Rejected'
if longpollid != 'init' or uri == config.longpoll_uri:
event = gevent.event.Event()
self.add_longpoll_event(event)
event.wait()
longpollid = self.get_work_id()
coinbase_tx = self.create_coinbase_tx(b'', b'')
block_template = {k: self.block_template[k]
for k in self.block_template
if k not in
['coinbasevalue', 'coinbaseaux', 'coinbaseflags']}
block_template['target'] = util.b2h(
self._get_target(auth['difficulty']))
block_template['mutable'] = ["coinbase/append", "submit/coinbase"]
block_template['transactions'] = [x.serialize() for x in self.tx]
block_template['coinbasetxn'] = coinbase_tx.serialize()
#Long polling extension
block_template['longpollid'] = longpollid
block_template['expires'] = 120
block_template['submitold'] = True
block_template['longpolluri'] = config.longpoll_uri
return block_template
def shutdown(self):
for agent_uuid in self.agents.keys():
_log.debug("Stopping agent UUID {}".format(agent_uuid))
self.stop_agent(agent_uuid)
event = gevent.event.Event()
agent = Agent(identity='aip', address='inproc://vip',
message_bus=self.message_bus)
task = gevent.spawn(agent.core.run, event)
try:
event.wait()
finally:
agent.core.stop()
task.kill()
def _get_rpc_agent(self, address):
if address == self.core.address:
agent = self
elif address not in self._vip_channels:
agent = Agent(address=address)
event = gevent.event.Event()
agent.core.onstart.connect(lambda *a, **kw: event.set(), event)
gevent.spawn(agent.core.run)
event.wait()
self._vip_channels[address] = agent
else:
agent = self._vip_channels[address]
return agent
def _get_rpc_agent(self, address):
if address == self.core.address:
agent = self
elif address not in self._vip_channels:
agent = Agent(address=address)
event = gevent.event.Event()
agent.core.onstart.connect(lambda *a, **kw: event.set(), event)
gevent.spawn(agent.core.run)
event.wait()
self._vip_channels[address] = agent
else:
agent = self._vip_channels[address]
return agent
def shutdown(self):
for agent_uuid in self.agents.iterkeys():
self.stop_agent(agent_uuid)
event = gevent.event.Event()
agent = Agent(identity='aip', address='inproc://vip')
task = gevent.spawn(agent.core.run, event)
try:
event.wait()
agent.vip.pubsub.publish(
'pubsub', topics.PLATFORM_SHUTDOWN,
{'reason': 'Received shutdown command'}).get()
finally:
agent.core.stop()
task.kill()
def shutdown(self):
for agent_uuid in self.agents.iterkeys():
self.stop_agent(agent_uuid)
event = gevent.event.Event()
agent = Agent(identity='aip', address='inproc://vip')
task = gevent.spawn(agent.core.run, event)
try:
event.wait()
agent.vip.pubsub.publish('pubsub', topics.PLATFORM_SHUTDOWN, {
'reason': 'Received shutdown command'
}).get()
finally:
agent.core.stop()
task.kill()
def stop(self, user_id, watch_users, timeout=None):
wait_id = self._get_wait_id()
event = gevent.event.Event()
self.waiting += 1
self.users.setdefault(user_id, {})[wait_id] = event
for watch in watch_users:
self.users_watch.setdefault(watch, {})[wait_id] = event
try:
event.wait(timeout)
finally:
for watch in watch_users:
del self.users_watch[watch][wait_id]
if len(self.users_watch[watch]) == 0:
del self.users_watch[watch]
del self.users[user_id][wait_id]
if len(self.users[user_id]) == 0:
del self.users[user_id]
self.waiting -= 1
def getwork(self, params, uri, auth):
if len(params) > 0:
block = util.h2b(params[0])
merkle_root = block[36:68]
if merkle_root not in self.work_data:
logger.error("Unknown worker submission")
return False
coinbase_tx = self.work_data[merkle_root].raw_tx
block += util.encode_size(len(self.tx) + 1)
block += coinbase_tx
result = self.process_block(block, auth)
del self.work_data[merkle_root]
return result
if uri == config.longpoll_uri:
event = gevent.event.Event()
self.add_longpoll_event(event)
event.wait()
coinbase_tx = self.create_coinbase_tx(util.h2b(self.get_work_id()),
b'')
merkle_root = MerkleTree.merkle_root_from_branch(
coinbase_tx.raw_tx, self.merkle_branches)
ntime = struct.pack(b'<I', self.block_template['curtime'])
block_header = self.create_block_header(merkle_root, ntime,
b'\x00\x00\x00\x00')
block_header += (b"\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x80")
self.work_data[merkle_root] = coinbase_tx
# To little endian
block_header = b''.join([
block_header[x:x + 4][::-1] for x in range(0, len(block_header), 4)
])
target = util.b2h(self._get_target(auth['difficulty'])[::-1])
block_header = util.b2h(block_header)
#TODO midstate, hash1 (deprecated)
return {'data': block_header, 'target': target}
def getwork(self, params, uri, auth):
if len(params) > 0:
block = util.h2b(params[0])
merkle_root = block[36:68]
if merkle_root not in self.work_data:
logger.error("Unknown worker submission")
return False
coinbase_tx = self.work_data[merkle_root].raw_tx
block += util.encode_size(len(self.tx) + 1)
block += coinbase_tx
result = self.process_block(block, auth)
del self.work_data[merkle_root]
return result
if uri == config.longpoll_uri:
event = gevent.event.Event()
self.add_longpoll_event(event)
event.wait()
coinbase_tx = self.create_coinbase_tx(
util.h2b(self.get_work_id()), b'')
merkle_root = MerkleTree.merkle_root_from_branch(
coinbase_tx.raw_tx, self.merkle_branches)
ntime = struct.pack(b'<I', self.block_template['curtime'])
block_header = self.create_block_header(merkle_root, ntime,
b'\x00\x00\x00\x00')
block_header += (b"\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x80")
self.work_data[merkle_root] = coinbase_tx
# To little endian
block_header = b''.join([block_header[x:x+4][::-1]
for x in range(0, len(block_header), 4)])
target = util.b2h(self._get_target(auth['difficulty'])[::-1])
block_header = util.b2h(block_header)
#TODO midstate, hash1 (deprecated)
return {'data': block_header,
'target': target}
def block_pusher(self):
try:
event = gevent.event.Event()
while True:
self._send_stratum_message(jsonrpc.create_request(
'mining.set_difficulty',
[self.auth['difficulty']]))
params = self.work.get_stratum_work(self.extranonce1)
self._send_stratum_message(jsonrpc.create_request(
'mining.notify', params))
self.last_work_id = params[0]
self.work.add_longpoll_event(event)
event.wait()
event.clear()
except:
logger.error("Pusher exception")
logger.error(traceback.format_exc())
def yield_future(future, loop=None):
"""Wait for a future, a task, or a coroutine object from a greenlet.
Yield control other eligible greenlet until the future is done (finished
successfully or failed with an exception).
Return the result or raise the exception of the future.
The function must not be called from the greenlet running the aiogreen
event loop.
"""
future = asyncio.async(future, loop=loop)
if future._loop._greenlet == gevent.getcurrent():
raise RuntimeError("yield_future() must not be called from "
"the greenlet of the aiogreen event loop")
event = gevent.event.Event()
def wakeup_event(fut):
event.set()
future.add_done_callback(wakeup_event)
event.wait()
return future.result()
def schedule_loop():
heap = self._schedule
event = self._schedule_event
cur = gevent.getcurrent()
now = time.time()
while True:
if heap:
deadline = heap[0][0]
timeout = min(5.0, max(0.0, deadline - now))
else:
timeout = None
if event.wait(timeout):
event.clear()
now = time.time()
while heap and now >= heap[0][0]:
_, callback = heapq.heappop(heap)
greenlet = gevent.spawn(callback)
cur.link(lambda glt: greenlet.kill())
def schedule_loop():
heap = self._schedule
event = self._schedule_event
cur = gevent.getcurrent()
now = time.time()
while True:
if heap:
deadline = heap[0][0]
timeout = min(5.0, max(0.0, deadline - now))
else:
timeout = None
if event.wait(timeout):
event.clear()
now = time.time()
while heap and now >= heap[0][0]:
_, callback = heapq.heappop(heap)
greenlet = gevent.spawn(callback)
cur.link(lambda glt: greenlet.kill())
def infinite(self):
event = gevent.event.Event()
event.wait()
def open(id):
# Wait for the next file to open.
while len(pending) == 0 or pending[0]['id'] < id:
event.wait(600)
return pending.pop(0)
def infinite(self):
event = gevent.event.Event()
event.wait()
def server(self):
if self._greenlet is None:
event = gevent.event.Event()
self._greenlet = gevent.spawn(self._server.core.run, event)
event.wait()
return self._server
def check(event):
delay = 0.0005
while not event.is_set():
event.wait(delay)
delay = min(delay * 2, .05)
def server(self):
if self._greenlet is None:
event = gevent.event.Event()
self._greenlet = gevent.spawn(self._server.core.run, event)
event.wait()
return self._server
def main():
opt_parser = optparse.OptionParser(usage="%prog [options] REQ_PER_SEC "
"SERVER_ADDR [SERVER_ADDR1...]")
opt_parser.add_option('--profile', dest="is_profile",
help="Turn on the profile", action="store_true",
default=False)
opt_parser.add_option('--cpu-affinity', dest="cpu_affinity",
help="Set the CPU affinity", type=int)
opt_parser.add_option('--runtime', dest="runtime",
help="Run specified time and exit", type=int)
opts, args = opt_parser.parse_args()
if len(args) < 2:
opt_parser.error('please enter REQ_PER_SEC SERVER_ADDR')
req_num = args[0]
try:
req_num = int(req_num)
except ValueError:
opt_parser.error('please enter correct REQ_PER_SEC')
server_addrs = args[1:]
print 'connecting to %s' % str(server_addrs)
logging.basicConfig(level=logging.DEBUG)
if opts.cpu_affinity is not None:
p = psutil.Process()
p.set_cpu_affinity([opts.cpu_affinity])
with greenprofile.Profiler(opts.is_profile, 'client.profile'):
client = recall.client.RpcClient()
channel = client.get_tcp_channel(server_addrs)
stub = test_pb2.TestService_Stub(channel)
event = gevent.event.Event()
def timer():
while True:
event.set()
gevent.sleep(1)
gevent.spawn(timer)
req = test_pb2.TestRequest(name='test_client', num=12345)
sent_req_num = 0
recv_rsp_num = [0]
def done(_, rsp):
if rsp:
recv_rsp_num[0] += 1
start_time = time.time()
try:
while True:
event.wait()
event.clear()
print 'sent_req_num', sent_req_num, 'recv_rsp_num', recv_rsp_num[0]
sent_req_num, recv_rsp_num[0] = 0, 0
for _ in xrange(req_num):
controller = RpcController()
stub.TestMethod(controller, req, done)
sent_req_num = req_num
if opts.runtime is not None:
now = time.time()
if now - start_time >= opts.runtime:
break
finally:
client.close()
def wait(what, timeout=30):
event = _get_or_make_event(what)
LOG.debug('waiting on %s' % what)
result = event.wait(timeout)
event.clear()
return result