Exemplo n.º 1
0
    def polling_app(environ, start_response):
        # If we're sitting behind an elb providing HTTPS:
        forwarded_proto = environ.get('HTTP_X_FORWARDED_PROTO')
        if forwarded_proto:
            environ['wsgi.url_scheme'] = forwarded_proto

        path = environ['PATH_INFO']
        if path.endswith('/longpoll'):
            client_gen = int(environ.get('HTTP_X_GENERATION', 0))
            if client_gen:
                zoid = int(environ['HTTP_X_GENERATION_ZOID'])
                event = gevent.event.Event()
                pollers[zoid].append(event)
                with db.transaction() as conn:
                    gen = conn.get(p64(zoid)).generation

                if gen <= client_gen:
                    # Wait for updates
                    event.wait(300)
                else:
                    try:
                        pollers[zoid].remove(event)
                    except IndexError:
                        pass

        return app(environ, start_response)
Exemplo n.º 2
0
def yield_future(future, loop=None):
    """Wait for a future, a task, or a coroutine object from a greenlet.

    Yield control other eligible greenlet until the future is done (finished
    successfully or failed with an exception).

    Return the result or raise the exception of the future.

    The function must not be called from the greenlet running the aiogreen
    event loop.
    """
    if hasattr(asyncio, "ensure_future"):
        ensure_future = asyncio.ensure_future
    else:  # use of async keyword has been Deprecated since Python 3.4.4
        ensure_future = getattr(asyncio, "async")
    future = ensure_future(future, loop=loop)
    if future._loop._greenlet == gevent.getcurrent():
        raise RuntimeError("yield_future() must not be called from "
                           "the greenlet of the aiogreen event loop")

    event = gevent.event.Event()

    def wakeup_event(fut):
        event.set()

    future.add_done_callback(wakeup_event)
    event.wait()
    return future.result()
Exemplo n.º 3
0
 def _get(self, client, timeout, count):
     if not timeout:
         timeout = 1
     exp = datetime.utcnow() + timedelta(seconds=timeout)
     messages = []
     event = gevent.event.Event()
     self.q.put((exp, count, client, messages, event))
     event.wait()
     return messages
Exemplo n.º 4
0
 def _get(self, client, timeout, count):
     if not timeout:
         timeout = 1
     exp = datetime.utcnow() + timedelta(seconds=timeout)
     messages = []
     event = gevent.event.Event()
     self.q.put((exp, count, client, messages, event))
     event.wait()
     return messages
Exemplo n.º 5
0
def callwhen(event, fun):
    while True:
        try:
            event.wait()
            event.clear()
            fun()
        except gevent.GreenletExit:
            raise
        except Exception:
            traceback.print_exc()
            pass
Exemplo n.º 6
0
    def getblocktemplate(self, params, uri, auth):
        """For worker"""

        longpollid = 'init'
        mode = 'template'  # For older client
        data = None
        for i in params:
            if 'longpollid' in i:
                longpollid = i['longpollid']
            if 'mode' in i:
                mode = i['mode']
            if 'data' in i:
                data = i['data']
        if mode == 'submit':
            try:
                block = util.h2b(data)
                sizelen, tx_size = util.decode_size(block[80:89])
                ptr = 80 + sizelen

                txlen = CoinbaseTransaction.verify(block[ptr:],
                                                   self.generation_pubkey)
                result = self.process_block(block[:ptr + txlen], auth)
                if result:
                    return None
            except:
                logger.error("Exception while processing block")
                logger.error(traceback.format_exc())
            return 'Rejected'

        if longpollid != 'init' or uri == config.longpoll_uri:
            event = gevent.event.Event()
            self.add_longpoll_event(event)
            event.wait()
            longpollid = self.get_work_id()
        coinbase_tx = self.create_coinbase_tx(b'', b'')
        block_template = {
            k: self.block_template[k]
            for k in self.block_template
            if k not in ['coinbasevalue', 'coinbaseaux', 'coinbaseflags']
        }

        block_template['target'] = util.b2h(
            self._get_target(auth['difficulty']))
        block_template['mutable'] = ["coinbase/append", "submit/coinbase"]
        block_template['transactions'] = [x.serialize() for x in self.tx]
        block_template['coinbasetxn'] = coinbase_tx.serialize()

        #Long polling extension
        block_template['longpollid'] = longpollid
        block_template['expires'] = 120
        block_template['submitold'] = True
        block_template['longpolluri'] = config.longpoll_uri

        return block_template
Exemplo n.º 7
0
    def getblocktemplate(self, params, uri, auth):
        """For worker"""

        longpollid = 'init'
        mode = 'template'  # For older client
        data = None
        for i in params:
            if 'longpollid' in i:
                longpollid = i['longpollid']
            if 'mode' in i:
                mode = i['mode']
            if 'data' in i:
                data = i['data']
        if mode == 'submit':
            try:
                block = util.h2b(data)
                sizelen, tx_size = util.decode_size(block[80:89])
                ptr = 80 + sizelen

                txlen = CoinbaseTransaction.verify(block[ptr:],
                                                   self.generation_pubkey)
                result = self.process_block(block[:ptr+txlen], auth)
                if result:
                    return None
            except:
                logger.error("Exception while processing block")
                logger.error(traceback.format_exc())
            return 'Rejected'

        if longpollid != 'init' or uri == config.longpoll_uri:
            event = gevent.event.Event()
            self.add_longpoll_event(event)
            event.wait()
            longpollid = self.get_work_id()
        coinbase_tx = self.create_coinbase_tx(b'', b'')
        block_template = {k: self.block_template[k]
                          for k in self.block_template
                          if k not in
                          ['coinbasevalue', 'coinbaseaux', 'coinbaseflags']}

        block_template['target'] = util.b2h(
            self._get_target(auth['difficulty']))
        block_template['mutable'] = ["coinbase/append", "submit/coinbase"]
        block_template['transactions'] = [x.serialize() for x in self.tx]
        block_template['coinbasetxn'] = coinbase_tx.serialize()

        #Long polling extension
        block_template['longpollid'] = longpollid
        block_template['expires'] = 120
        block_template['submitold'] = True
        block_template['longpolluri'] = config.longpoll_uri

        return block_template
Exemplo n.º 8
0
 def shutdown(self):
     for agent_uuid in self.agents.keys():
         _log.debug("Stopping agent UUID {}".format(agent_uuid))
         self.stop_agent(agent_uuid)
     event = gevent.event.Event()
     agent = Agent(identity='aip', address='inproc://vip',
                   message_bus=self.message_bus)
     task = gevent.spawn(agent.core.run, event)
     try:
         event.wait()
     finally:
         agent.core.stop()
         task.kill()
Exemplo n.º 9
0
        def _get_rpc_agent(self, address):
            if address == self.core.address:
                agent = self
            elif address not in self._vip_channels:
                agent = Agent(address=address)
                event = gevent.event.Event()
                agent.core.onstart.connect(lambda *a, **kw: event.set(), event)
                gevent.spawn(agent.core.run)
                event.wait()
                self._vip_channels[address] = agent

            else:
                agent = self._vip_channels[address]
            return agent
Exemplo n.º 10
0
        def _get_rpc_agent(self, address):
            if address == self.core.address:
                agent = self
            elif address not in self._vip_channels:
                agent = Agent(address=address)
                event = gevent.event.Event()
                agent.core.onstart.connect(lambda *a, **kw: event.set(), event)
                gevent.spawn(agent.core.run)
                event.wait()
                self._vip_channels[address] = agent

            else:
                agent = self._vip_channels[address]
            return agent
Exemplo n.º 11
0
 def shutdown(self):
     for agent_uuid in self.agents.iterkeys():
         self.stop_agent(agent_uuid)
     event = gevent.event.Event()
     agent = Agent(identity='aip', address='inproc://vip')
     task = gevent.spawn(agent.core.run, event)
     try:
         event.wait()
         agent.vip.pubsub.publish(
             'pubsub', topics.PLATFORM_SHUTDOWN,
             {'reason': 'Received shutdown command'}).get()
     finally:
         agent.core.stop()
         task.kill()
Exemplo n.º 12
0
 def shutdown(self):
     for agent_uuid in self.agents.iterkeys():
         self.stop_agent(agent_uuid)
     event = gevent.event.Event()
     agent = Agent(identity='aip', address='inproc://vip')
     task = gevent.spawn(agent.core.run, event)
     try:
         event.wait()
         agent.vip.pubsub.publish('pubsub', topics.PLATFORM_SHUTDOWN, {
             'reason': 'Received shutdown command'
         }).get()
     finally:
         agent.core.stop()
         task.kill()
Exemplo n.º 13
0
 def stop(self, user_id, watch_users, timeout=None):
     wait_id = self._get_wait_id()
     event = gevent.event.Event()
     self.waiting += 1
     self.users.setdefault(user_id, {})[wait_id] = event
     for watch in watch_users:
         self.users_watch.setdefault(watch, {})[wait_id] = event
     try:
         event.wait(timeout)
     finally:
         for watch in watch_users:
             del self.users_watch[watch][wait_id]
             if len(self.users_watch[watch]) == 0:
                 del self.users_watch[watch]
         del self.users[user_id][wait_id]
         if len(self.users[user_id]) == 0:
             del self.users[user_id]
         self.waiting -= 1
Exemplo n.º 14
0
    def getwork(self, params, uri, auth):
        if len(params) > 0:
            block = util.h2b(params[0])
            merkle_root = block[36:68]

            if merkle_root not in self.work_data:
                logger.error("Unknown worker submission")
                return False
            coinbase_tx = self.work_data[merkle_root].raw_tx
            block += util.encode_size(len(self.tx) + 1)
            block += coinbase_tx

            result = self.process_block(block, auth)

            del self.work_data[merkle_root]
            return result

        if uri == config.longpoll_uri:
            event = gevent.event.Event()
            self.add_longpoll_event(event)
            event.wait()

        coinbase_tx = self.create_coinbase_tx(util.h2b(self.get_work_id()),
                                              b'')
        merkle_root = MerkleTree.merkle_root_from_branch(
            coinbase_tx.raw_tx, self.merkle_branches)
        ntime = struct.pack(b'<I', self.block_template['curtime'])
        block_header = self.create_block_header(merkle_root, ntime,
                                                b'\x00\x00\x00\x00')
        block_header += (b"\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
                         b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
                         b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
                         b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x80")

        self.work_data[merkle_root] = coinbase_tx

        # To little endian
        block_header = b''.join([
            block_header[x:x + 4][::-1] for x in range(0, len(block_header), 4)
        ])
        target = util.b2h(self._get_target(auth['difficulty'])[::-1])
        block_header = util.b2h(block_header)
        #TODO midstate, hash1 (deprecated)
        return {'data': block_header, 'target': target}
Exemplo n.º 15
0
    def getwork(self, params, uri, auth):
        if len(params) > 0:
            block = util.h2b(params[0])
            merkle_root = block[36:68]

            if merkle_root not in self.work_data:
                logger.error("Unknown worker submission")
                return False
            coinbase_tx = self.work_data[merkle_root].raw_tx
            block += util.encode_size(len(self.tx) + 1)
            block += coinbase_tx

            result = self.process_block(block, auth)

            del self.work_data[merkle_root]
            return result

        if uri == config.longpoll_uri:
            event = gevent.event.Event()
            self.add_longpoll_event(event)
            event.wait()

        coinbase_tx = self.create_coinbase_tx(
            util.h2b(self.get_work_id()), b'')
        merkle_root = MerkleTree.merkle_root_from_branch(
            coinbase_tx.raw_tx, self.merkle_branches)
        ntime = struct.pack(b'<I', self.block_template['curtime'])
        block_header = self.create_block_header(merkle_root, ntime,
                                                b'\x00\x00\x00\x00')
        block_header += (b"\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
                         b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
                         b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
                         b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x80")

        self.work_data[merkle_root] = coinbase_tx

        # To little endian
        block_header = b''.join([block_header[x:x+4][::-1]
                                for x in range(0, len(block_header), 4)])
        target = util.b2h(self._get_target(auth['difficulty'])[::-1])
        block_header = util.b2h(block_header)
        #TODO midstate, hash1 (deprecated)
        return {'data': block_header,
                'target': target}
Exemplo n.º 16
0
    def block_pusher(self):
        try:
            event = gevent.event.Event()
            while True:
                self._send_stratum_message(jsonrpc.create_request(
                                           'mining.set_difficulty',
                                           [self.auth['difficulty']]))

                params = self.work.get_stratum_work(self.extranonce1)
                self._send_stratum_message(jsonrpc.create_request(
                                           'mining.notify', params))

                self.last_work_id = params[0]
                self.work.add_longpoll_event(event)
                event.wait()
                event.clear()
        except:
            logger.error("Pusher exception")
            logger.error(traceback.format_exc())
Exemplo n.º 17
0
def yield_future(future, loop=None):
    """Wait for a future, a task, or a coroutine object from a greenlet.

    Yield control other eligible greenlet until the future is done (finished
    successfully or failed with an exception).

    Return the result or raise the exception of the future.

    The function must not be called from the greenlet running the aiogreen
    event loop.
    """
    future = asyncio.async(future, loop=loop)
    if future._loop._greenlet == gevent.getcurrent():
        raise RuntimeError("yield_future() must not be called from "
                           "the greenlet of the aiogreen event loop")

    event = gevent.event.Event()

    def wakeup_event(fut):
        event.set()

    future.add_done_callback(wakeup_event)
    event.wait()
    return future.result()
Exemplo n.º 18
0
 def schedule_loop():
     heap = self._schedule
     event = self._schedule_event
     cur = gevent.getcurrent()
     now = time.time()
     while True:
         if heap:
             deadline = heap[0][0]
             timeout = min(5.0, max(0.0, deadline - now))
         else:
             timeout = None
         if event.wait(timeout):
             event.clear()
         now = time.time()
         while heap and now >= heap[0][0]:
             _, callback = heapq.heappop(heap)
             greenlet = gevent.spawn(callback)
             cur.link(lambda glt: greenlet.kill())
Exemplo n.º 19
0
 def schedule_loop():
     heap = self._schedule
     event = self._schedule_event
     cur = gevent.getcurrent()
     now = time.time()
     while True:
         if heap:
             deadline = heap[0][0]
             timeout = min(5.0, max(0.0, deadline - now))
         else:
             timeout = None
         if event.wait(timeout):
             event.clear()
         now = time.time()
         while heap and now >= heap[0][0]:
             _, callback = heapq.heappop(heap)
             greenlet = gevent.spawn(callback)
             cur.link(lambda glt: greenlet.kill())
Exemplo n.º 20
0
 def infinite(self):
     event = gevent.event.Event()
     event.wait()
Exemplo n.º 21
0
def open(id):
    # Wait for the next file to open.
    while len(pending) == 0 or pending[0]['id'] < id:
        event.wait(600)
    return pending.pop(0)
Exemplo n.º 22
0
 def infinite(self):
     event = gevent.event.Event()
     event.wait()
Exemplo n.º 23
0
 def server(self):
     if self._greenlet is None:
         event = gevent.event.Event()
         self._greenlet = gevent.spawn(self._server.core.run, event)
         event.wait()
     return self._server
Exemplo n.º 24
0
 def check(event):
     delay = 0.0005
     while not event.is_set():
         event.wait(delay)
         delay = min(delay * 2, .05)
Exemplo n.º 25
0
 def server(self):
     if self._greenlet is None:
         event = gevent.event.Event()
         self._greenlet = gevent.spawn(self._server.core.run, event)
         event.wait()
     return self._server
Exemplo n.º 26
0
def main():
    opt_parser = optparse.OptionParser(usage="%prog [options] REQ_PER_SEC "
                                       "SERVER_ADDR [SERVER_ADDR1...]")
    opt_parser.add_option('--profile', dest="is_profile",
                          help="Turn on the profile", action="store_true",
                          default=False)
    opt_parser.add_option('--cpu-affinity', dest="cpu_affinity",
                          help="Set the CPU affinity", type=int)
    opt_parser.add_option('--runtime', dest="runtime",
                          help="Run specified time and exit", type=int)

    opts, args = opt_parser.parse_args()

    if len(args) < 2:
        opt_parser.error('please enter REQ_PER_SEC SERVER_ADDR')

    req_num = args[0]
    try:
        req_num = int(req_num)
    except ValueError:
        opt_parser.error('please enter correct REQ_PER_SEC')

    server_addrs = args[1:]

    print 'connecting to %s' % str(server_addrs)

    logging.basicConfig(level=logging.DEBUG)

    if opts.cpu_affinity is not None:
        p = psutil.Process()
        p.set_cpu_affinity([opts.cpu_affinity])

    with greenprofile.Profiler(opts.is_profile, 'client.profile'):
        client = recall.client.RpcClient()
        channel = client.get_tcp_channel(server_addrs)
        stub = test_pb2.TestService_Stub(channel)

        event = gevent.event.Event()

        def timer():
            while True:
                event.set()
                gevent.sleep(1)

        gevent.spawn(timer)

        req = test_pb2.TestRequest(name='test_client', num=12345)
        sent_req_num = 0
        recv_rsp_num = [0]

        def done(_, rsp):
            if rsp:
                recv_rsp_num[0] += 1

        start_time = time.time()
        try:
            while True:
                event.wait()
                event.clear()
                print 'sent_req_num', sent_req_num, 'recv_rsp_num', recv_rsp_num[0]
                sent_req_num, recv_rsp_num[0] = 0, 0

                for _ in xrange(req_num):
                    controller = RpcController()
                    stub.TestMethod(controller, req, done)

                sent_req_num = req_num
                if opts.runtime is not None:
                    now = time.time()
                    if now - start_time >= opts.runtime:
                        break

        finally:
            client.close()
Exemplo n.º 27
0
def wait(what, timeout=30):
    event = _get_or_make_event(what)
    LOG.debug('waiting on %s' % what)
    result = event.wait(timeout)
    event.clear()
    return result