class CexGetOrderBookTask(object): def __init__(self, injector, target): self.client = injector.cex_client self.channels = injector.channels self.target = target self.async_result = AsyncResult() def execute(self): gevent.spawn(self.get_orderbook) return self.async_result def get_orderbook(self): try: _, equity_pair = self.target.orderbook_key.split(':') channel = self.channels.get_channel_or_none('cex') orderbook = None if channel is not None: # 만약 Channel이 있다면 Channel을 통해 Orderbook을 가져와본다. orderbook = channel.get_orderbook_or_none(equity_pair=equity_pair, depth=20) if orderbook is None: # 만약 Channel을 통해 Orderbook을 가져오지 못했다면 HTTP로 시도해본다. orderbook = self.client.get_orderbook(equity_pair=equity_pair, depth=20) self.async_result.set(orderbook) except Exception as e: self.async_result.set_exception(e)
class OkexGetOrderBookTask(object): def __init__(self, injector, target): self.client = injector.okex_client self.logger = injector.logger self.target = target self.async_result = AsyncResult() def execute(self): gevent.spawn(self.get_orderbook) return self.async_result def get_orderbook(self): try: if self.target.type == 'spot': _, equity_pair = self.target.orderbook_key.split(':') orderbook = self.client.get_spot_orderbook( equity_pair=equity_pair) self.async_result.set(orderbook) elif self.target.type == 'future': equity_pair = self.target.equity_pair contract_type = self.target.contract_type orderbook = self.client.get_future_orderbook( equity_pair=equity_pair, contract_type=contract_type) self.async_result.set(orderbook) except Exception as e: self.async_result.set_exception(e)
class DAOChallenger(object): request_timeout = 8. def __init__(self, chainservice, proto): self.chainservice = chainservice self.config = chainservice.config['eth']['block'] self.proto = proto self.deferred = None gevent.spawn(self.run) def run(self): self.deferred = AsyncResult() self.proto.send_getblockheaders(self.config['DAO_FORK_BLKNUM'], 1, 0) try: dao_headers = self.deferred.get(block=True, timeout=self.request_timeout) log.debug("received DAO challenge answer", proto=self.proto, answer=dao_headers) result = len(dao_headers) == 1 and \ dao_headers[0].hash == self.config['DAO_FORK_BLKHASH'] and \ dao_headers[0].extra_data == self.config['DAO_FORK_BLKEXTRA'] self.chainservice.on_dao_challenge_answer(self.proto, result) except gevent.Timeout: log.debug('challenge dao timed out', proto=self.proto) self.chainservice.on_dao_challenge_answer(self.proto, False) def receive_blockheaders(self, proto, blockheaders): log.debug('blockheaders received', proto=proto, num=len(blockheaders)) if proto != self.proto: return self.deferred.set(blockheaders)
def send_async( self, receiver_address: Address, queue_name: bytes, message: Message, ): if not self._running: return self.log.info( 'SEND ASYNC', receiver_address=to_normalized_address(receiver_address), message=message, queue_name=queue_name, ) if not is_binary_address(receiver_address): raise ValueError('Invalid address {}'.format( pex(receiver_address))) # These are not protocol messages, but transport specific messages if isinstance(message, (Delivered, Ping, Pong)): raise ValueError( 'Do not use send_async for {} messages'.format( message.__class__.__name__), ) message_id = message.message_identifier async_result = AsyncResult() if isinstance(message, Processed): async_result.set( True) # processed messages shouldn't get a Delivered reply self._send_immediate(receiver_address, json.dumps(message.to_dict())) else: self._messageids_to_asyncresult[message_id] = async_result self._send_with_retry(receiver_address, async_result, json.dumps(message.to_dict()))
def register_secret(self, secret: typing.Secret): secrethash = sha3(secret) if self.check_registered(secrethash): log.info( 'secret already registered', node=pex(self.node_address), contract=pex(self.address), secrethash=encode_hex(secrethash), ) return log.info( 'registerSecret called', node=pex(self.node_address), contract=pex(self.address), ) if secret not in self.open_secret_transactions: secret_registry_transaction = AsyncResult() self.open_secret_transactions[secret] = secret_registry_transaction try: transaction_hash = self._register_secret(secret) except Exception as e: secret_registry_transaction.set_exception(e) raise else: secret_registry_transaction.set(transaction_hash) finally: self.open_secret_transactions.pop(secret, None) else: transaction_hash = self.open_secret_transactions[secret].get()
def test__control_flow_cancelled_call(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) self.addCleanup(p.stop) # put a call in that will never finish waitar = AsyncResult() # test specific, wait for this to indicate we're being processed/hung callar = AsyncResult() # test specific, an ar that is just waited on by the spin call (eventually set in this test) def spin(inar, outar): outar.set(True) inar.wait() ar = p._routing_call(spin, MagicMock(), callar, waitar) # schedule a second call that we're going to cancel futurear = AsyncResult() ar2 = p._routing_call(futurear.set, MagicMock(), sentinel.val) # wait until we get notice we're being processed waitar.get(timeout=2) # cancel the SECOND call p.cancel_or_abort_call(ar2) # prove we didn't interrupt the current proc by allowing it to continue callar.set() ar.get(timeout=2) # now the second proc will get queued and never called because it is cancelled self.assertRaises(Timeout, futurear.get, timeout=2) self.assertTrue(ar2.ready())
class InstrumentAgentEventSubscribers(object): """ Create subscribers for agent and driver events. """ log.info("Start event subscribers") def __init__(self, instrument_agent_resource_id = None): # Start event subscribers, add stop to cleanup. self.no_events = None self.events_received = [] self.async_event_result = AsyncResult() self.event_subscribers = [] def consume_event(*args, **kwargs): log.debug('#**#**# Event subscriber (consume_event) recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) log.debug("self.no_events = " + str(self.no_events)) log.debug("self.event_received = " + str(self.events_received)) self.events_received.append(args[0]) if self.no_events and self.no_events == len(self.events_received): log.debug("CALLING self.async_event_result.set()") self.async_event_result.set() self.event_subscribers = EventSubscriber( event_type='ResourceAgentEvent', callback=consume_event, origin=instrument_agent_resource_id) self.event_subscribers.start() self.event_subscribers._ready_event.wait(timeout=5)
def __init__(self, rpc_destination, s_stomp, timeout=3600, init_timeout=None): """ Initialize an RpcProxy with a remote RpcProvider at rpc_destination through a SynchronousStomp object (s_stomp). If timeout is not None (default one hour) if any conversation blocks for that amount of time before receiving a reply, the call will raise a gevent.Timeout. """ self._rpc_destination = rpc_destination self._s_stomp = s_stomp self._services = {} self._class = None self._timeout = timeout # Initialize the alternate proxy table self_result = AsyncResult() self_result.set(self) self._timeout_proxies = {timeout: self_result} # n.b. After this point, no new attributes may be added conv = self._make_conversation() msg = self._build_rpc_request('.list_services') # This will use self._timeout if init_timeout is None reply = conv.reply(msg, timeout=init_timeout) # Store the new representation self._services = reply['services'] self._docs = reply['services_doc'] self._class = reply.get('name', None)
def fake_async(obj): """ For mocking RPC functions which will be called async """ asr = AsyncResult() asr.set(obj) return asr
def send_async( self, receiver_address: Address, queue_name: bytes, message: Message, ): if not self._running: return self.log.info( 'SEND ASYNC', receiver_address=to_normalized_address(receiver_address), message=message, queue_name=queue_name, ) if not is_binary_address(receiver_address): raise ValueError('Invalid address {}'.format(pex(receiver_address))) # These are not protocol messages, but transport specific messages if isinstance(message, (Delivered, Ping, Pong)): raise ValueError( 'Do not use send_async for {} messages'.format(message.__class__.__name__), ) message_id = message.message_identifier async_result = AsyncResult() if isinstance(message, Processed): async_result.set(True) # processed messages shouldn't get a Delivered reply self._send_immediate(receiver_address, json.dumps(message.to_dict())) else: self._messageids_to_asyncresult[message_id] = async_result self._send_with_retry(receiver_address, async_result, json.dumps(message.to_dict()))
def fake_async(obj): """ For mocking RPC functions which will be called async """ asr = AsyncResult() asr.set(obj) return asr
class LogListenerTask(Task): def __init__(self, filter_, callback, contract_translator): super(LogListenerTask, self).__init__() self.filter_ = filter_ self.callback = callback self.contract_translator = contract_translator self.stop_event = AsyncResult() self.sleep_time = 0.5 def _run(self): # pylint: disable=method-hidden stop = None while stop is None: filter_changes = self.filter_.changes() for log_event in filter_changes: event = self.contract_translator.decode_event( log_event['topics'], log_event['data'], ) if event is not None: originating_contract = log_event['address'] self.callback(originating_contract, event) stop = self.stop_event.wait(self.sleep_time) def stop(self): self.stop_event.set(True)
class LogListenerTask(Task): def __init__(self, filter_, callback, contract_translator): super(LogListenerTask, self).__init__() self.filter_ = filter_ self.callback = callback self.contract_translator = contract_translator self.stop_event = AsyncResult() self.sleep_time = 0.5 def _run(self): # pylint: disable=method-hidden stop = None while stop is None: filter_changes = self.filter_.changes() for log_event in filter_changes: event = self.contract_translator.decode_event( log_event['topics'], log_event['data'], ) if event is not None: originating_contract = log_event['address'] self.callback(originating_contract, event) stop = self.stop_event.wait(self.sleep_time) def stop(self): self.stop_event.set(True)
class TestAsyncResult(object): def __init__(self): self.event = AsyncResult() def run(self): producers = [gevent.spawn(self._producer, i) for i in xrange(3)] consumers = [gevent.spawn(self._consumer, i) for i in xrange(3)] tasks = [] tasks.extend(producers) tasks.extend(consumers) gevent.joinall(tasks) def _producer(self, pid): print("I'm producer %d and now I don't want consume to do something" % (pid,)) sleeptime = random.randint(5, 10) * 0.01 print("Sleeping time is %f" % (sleeptime, )) gevent.sleep(sleeptime) print("I'm producer %d and now consumer could do something." % (pid,)) self.event.set('producer pid %d' % (pid, )) def _consumer(self, pid): print("I'm consumer %d and now I'm waiting for producer" % (pid,)) gevent.sleep(random.randint(0, 5) * 0.01) value = self.event.wait() print("I'm consumer %d. Value is %r and now I can do something" % (pid, value))
def pozt(self, LocalMod): rvData = "" try: data = LocalMod["json"] dataEncoded = json.dumps(data) #req = self.http.request('POST', self.JITHashServer, body=dataEncoded) req = self.http.urlopen('POST', self.JITHashServer, headers=self.headers, body=dataEncoded) #response = self.http.urlopen(req) #rvData = req.data except HTTPError as inst: if inst.code == 204: return rvData except: print("{}{}".format(fg("red"), "SERVER FAILED DESPITE MULTIPLE ATTEMPTS")) print("{}{}{}[{}]".format(fg("navajo_white_1"), "Exception ", fg("light_magenta"), str(sys.exc_info()[0]))) for x in sys.exc_info(): print("{}{}".format(fg("hot_pink_1b"), x)) finally: a = AsyncResult() a.set(req.data) LocalMod["resp"] = a.get(block=True) req.release_conn() self.output(LocalMod) return LocalMod
def _event(): # 协程通信 e = Event() ae = AsyncResult() e.wait() # 唤醒 e.set() # 唤醒 ae.wait() # 唤醒 ae.set('') # 唤醒,可传递信息
class WSGIServer(pywsgi.WSGIServer): def __init__(self, app): pywsgi.WSGIServer.__init__(self, ("127.0.0.1", 0), app) self.localport = AsyncResult() def start_accepting(self): self.localport.set(self.socket.getsockname()[1]) pywsgi.WSGIServer.start_accepting(self)
class WSGIServer(pywsgi.WSGIServer): def __init__(self, app): pywsgi.WSGIServer.__init__(self, ("127.0.0.1", 0), app) self.localport = AsyncResult() def start_accepting(self): self.localport.set(self.socket.getsockname()[1]) pywsgi.WSGIServer.start_accepting(self)
class GConnection(Async): def __init__(self, *args, **kwargs): """ This class is a 'GEvent'-optimized subclass of libcouchbase which utilizes the underlying IOPS structures and the gevent event primitives to efficiently utilize couroutine switching. """ super(GConnection, self).__init__(IOPS(), *args, **kwargs) def _do_ctor_connect(self): if self.connected: return self._connect() self._evconn = AsyncResult() self._conncb = self._on_connected self._evconn.get() self._evconn = None def _on_connected(self, err): if err: self._evconn.set_exception(err) else: self._evconn.set(None) def _waitwrap(self, cbasync): cur_thread = getcurrent() cbasync.callback = cur_thread.switch cbasync.errback = lambda r, x, y, z: cur_thread.throw(x, y, z) return get_hub().switch() def _meth_factory(meth, name): def ret(self, *args, **kwargs): return self._waitwrap(meth(self, *args, **kwargs)) return ret def _http_request(self, **kwargs): res = super(GConnection, self)._http_request(**kwargs) if kwargs.get('chunked', False): return res #views e = Event() res._callback = lambda x, y: e.set() e.wait() res._maybe_raise() return res def query(self, *args, **kwargs): kwargs['itercls'] = GView ret = super(GConnection, self).query(*args, **kwargs) ret.start() return ret locals().update(Async._gen_memd_wrappers(_meth_factory))
class GConnection(Async): def __init__(self, *args, **kwargs): """ This class is a 'GEvent'-optimized subclass of libcouchbase which utilizes the underlying IOPS structures and the gevent event primitives to efficiently utilize couroutine switching. """ experimental.enabled_or_raise() super(GConnection, self).__init__(IOPS(), *args, **kwargs) def _do_ctor_connect(self): if self.connected: return self._connect() self._evconn = AsyncResult() self._conncb = self._on_connected self._evconn.get() self._evconn = None def _on_connected(self, err): if err: self._evconn.set_exception(err) else: self._evconn.set(None) def _waitwrap(self, cbasync): cur_thread = getcurrent() cbasync.callback = cur_thread.switch cbasync.errback = lambda r, x, y, z: cur_thread.throw(x, y, z) return get_hub().switch() def _meth_factory(meth, name): def ret(self, *args, **kwargs): return self._waitwrap(meth(self, *args, **kwargs)) return ret def _http_request(self, **kwargs): res = super(GConnection, self)._http_request(**kwargs) if kwargs.get('chunked', False): return res #views e = Event() res._callback = lambda x, y: e.set() e.wait() res._maybe_raise() return res def query(self, *args, **kwargs): kwargs['itercls'] = GView ret = super(GConnection, self).query(*args, **kwargs) ret.start() return ret locals().update(Async._gen_memd_wrappers(_meth_factory))
def get_dirty_values_async_result(self): if self.mode == 'r': log.warn('Coverage not open for writing: mode=%s', self.mode) from gevent.event import AsyncResult ret = AsyncResult() ret.set(True) return ret return self._persistence_layer.get_dirty_values_async_result()
class DiskSubscription(object): """A Subscription used by a PubSub channel backed by DiskCache Attributes: channel (str): The channel name associated with this subscription _listener (generator): The generator which yields the new values in the channel _result (AsyncResult): The result of the latest value in the channel _sync (Event): A synchronization event used to block the channel from yielding new values Args: channel (str): The channel name associated with this subscription """ def __init__(self, channel): self.channel = channel self._listener = None self._result = AsyncResult() self._sync = Event() def listen(self): """Listen for updates in this channel Returns: (generator): A generator which yields new values in the channel """ if self._listener is None: self._listener = self._listen() return self._listener def _listen(self): """Listen for updates in this channel and yield the results Yields: The new values in this channel """ while True: self._sync.wait() result = self._result.get() if result == unsubscribe_message: break else: yield result sleep(0) def push(self, value): """Push a new value to the channel Args: value: The value to push to the channel """ if self._listener is not None: self._result.set(value) sleep(0) self._sync.set() sleep(0) self._sync.clear()
def handshake(self): """returns None on error or (node_host, node_port) """ data = {} data["name"] = base64.standard_b64encode( login.encrypt('frontend', socket.gethostname())) data["system"] = platform.system() data["machine"] = platform.machine() data["platform"] = platform.platform() data["release"] = platform.release() data["version"] = platform.version() result = AsyncResult() rid = str(id(result)) self.login_results[rid] = result key = login.generate_backend_key() from .. import patch payload = { 'id': rid, 'version': proto.VERSION, 'branch': patch.config.branch, 'commit_id': patch.core_source.version, 'l': login.get('login'), 'system': data, } message_key = proto.pack_message('backend', 'api.set_key', payload=dict(key=key), encrypt="rsa") message = proto.pack_message('backend', 'api.login', payload=payload) try: self.send_message(message_key) self.send_message(message) except AttributeError: result.set([False, 'Client login error']) return try: result = result.get(timeout=20) except gevent.Timeout: result = ["False", "Login timed out"] finally: try: del self.login_results[rid] except KeyError: pass if not result[0]: log.error('login failed: {}'.format(result[1])) if result[1] == 'Invalid Login Credentials': self.connect_retry = 0 login.logout() return False return True
def exchange_async(self, type_, base_amount, quote_amount, self_address, target_address, identifier): result_async = AsyncResult() key = hash(str(type_) + str(base_amount) + str(quote_amount) + str(self_address) + str(target_address) + str( identifier)) if key in self.expected: self.expected[key][0].set(True) result_async.set(True) del self.expected[key] else: result_async.set(False) return result_async
def wrapper(*args, **kwargs): if not ThreadPool.isMainThread(): return ThreadPool.main_loop.call(wrapper, *args, **kwargs) if self.ignore_class: key = func # Unique key only by function and class object elif self.ignore_args: key = (func, args[0] ) # Unique key only by function and class object else: key = (func, tuple(args), str(kwargs) ) # Unique key for function including parameters if key in self.threads: # Thread already running (if using blocking mode) if self.queue: self.queued = True thread = self.threads[key] if self.blocking: if self.queued: res = thread.get() # Blocking until its finished if key in self.threads: return self.threads[key].get( ) # Queue finished since started running self.queued = False return wrapper(*args, **kwargs) # Run again after the end else: return thread.get() # Return the value else: # No blocking if thread.ready(): # Its finished, create a new thread = gevent.spawn(func, *args, **kwargs) self.threads[key] = thread return thread else: # Still running return thread else: # Thread not running if self.blocking: # Wait for finish asyncres = AsyncResult() self.threads[key] = asyncres try: res = func(*args, **kwargs) asyncres.set(res) self.cleanup(key, asyncres) return res except Exception as err: asyncres.set_exception(err) self.cleanup(key, asyncres) raise (err) else: # No blocking just return the thread thread = gevent.spawn(func, *args, **kwargs) # Spawning new thread thread.link(lambda thread: self.cleanup(key, thread)) self.threads[key] = thread return thread
class DataMonitor(object): _STOP_REQUEST = object() def __init__(self, client, path, callback, args, kwargs): self.client = client self.path = path self.callback = callback self.args = args self.kwargs = kwargs self.started = AsyncResult() self.queue = Queue() self._delay = 1.343 self.max_delay = 180 def _monitor(self): """Run the monitoring loop.""" def watcher(event): self.queue.put(event) while True: try: data, stat = self.client.get(self.path, watcher) except zookeeper.NoNodeException: if not self.started.ready(): self.started.set(None) gevent.sleep(1) continue except (zookeeper.ConnectionLossException, zookeeper.SessionExpiredException, zookeeper.InvalidStateException), err: if not self.started.ready(): self.started.set_exception(err) break logging.error("got %r while monitoring %s", str(err), self.path) gevent.sleep(self._delay) self._delay += self._delay * random.random() self._delay = min(self._delay, self.max_delay) continue except Exception, err: if not self.started.ready(): self.started.set_exception(err) break raise self.callback(data, *self.args, **self.kwargs) if not self.started.ready(): self.started.set(None) event = self.queue.get() if event is self._STOP_REQUEST: break
class AsyncResultWaiter(object): """ Class that makes waiting for an async result notification easy. Creates a subscriber for a generated token name, which can be handed to the async provider. The provider then publishes the result to the token name when ready. The caller can wait for the result or timeout. """ def __init__(self, process=None): self.process = process self.async_res = AsyncResult() self.wait_name = "asyncresult_" + create_simple_unique_id() if self.process: self.wait_name = self.wait_name + "_" + self.process.id # TODO: Use same mechanism as pooled RPC response endpoint (without the request) self.wait_sub = Subscriber(from_name=self.wait_name, callback=self._result_callback, auto_delete=True) self.activated = False def activate(self): if self.activated: raise BadRequest("Already active") self.listen_gl = spawn(self.wait_sub.listen ) # This initializes and activates the listener self.wait_sub.get_ready_event().wait(timeout=1) self.activated = True return self.wait_name def _result_callback(self, msg, headers): log.debug("AsyncResultWaiter: received message") self.async_res.set(msg) def await (self, timeout=None, request_id=None): try: result = self.async_res.get(timeout=timeout) if request_id and isinstance( result, AsyncResultMsg) and result.request_id != request_id: log.warn("Received result for different request: %s", result) result = None except gevent.Timeout: raise Timeout("Timeout in AsyncResultWaiter name={}".format( self.wait_name)) finally: self.wait_sub.deactivate() self.wait_sub.close() self.listen_gl.join(timeout=1) self.activated = False return result
class ReadyCounter(object): def __init__(self): self.count = 0 self.event = AsyncResult() def add(self): self.count += 1 def ready(self): self.count -= 1 if self.count == 0: self.event.set()
class DataMonitor(object): _STOP_REQUEST = object() def __init__(self, client, path, callback, args, kwargs): self.client = client self.path = path self.callback = callback self.args = args self.kwargs = kwargs self.started = AsyncResult() self.queue = Queue() self._delay = 1.343 self.max_delay = 180 def _monitor(self): """Run the monitoring loop.""" def watcher(event): self.queue.put(event) while True: try: data, stat = self.client.get(self.path, watcher) except zookeeper.NoNodeException: if not self.started.ready(): self.started.set(None) gevent.sleep(1) continue except (zookeeper.ConnectionLossException, zookeeper.SessionExpiredException, zookeeper.InvalidStateException), err: if not self.started.ready(): self.started.set_exception(err) break logging.error("got %r while monitoring %s", str(err), self.path) gevent.sleep(self._delay) self._delay += self._delay * random.random() self._delay = min(self._delay, self.max_delay) continue except Exception, err: if not self.started.ready(): self.started.set_exception(err) break raise self.callback(data, *self.args, **self.kwargs) if not self.started.ready(): self.started.set(None) event = self.queue.get() if event is self._STOP_REQUEST: break
class ESCostCollector(object): def __init__(self): self.accumulated = 0 self.price = AsyncResult() def add(self, result): self.accumulated += result['took'] def finalize(self): self.price.set(self.accumulated) def get_price(self): return self.price.get()
def test_server_rejects_bad_version_info(self): server_done = AsyncResult() client = self.client_cxn def server_thread(): socket, addr = self.server_socket.accept() server = ServerConnection(socket, addr) try: server.recv_message() raise AssertionError("expected exception not raised") except ConnectionError, e: assert_contains(str(e), "VERSION_MISMATCH") assert not server.connected() server_done.set(True)
def test_server_rejects_bad_version_info(self): server_done = AsyncResult() client = self.client_cxn def server_thread(): socket, addr = self.server_socket.accept() server = ServerConnection(socket, addr) try: server.recv_message() raise AssertionError("expected exception not raised") except ConnectionError, e: assert_contains(str(e), "VERSION_MISMATCH") assert not server.connected() server_done.set(True)
class PageProxy(object): """A page in a wiki The page may not be loaded when this object is created; accessing its attributes may block. A page is true in a boolean context if it exists on the wiki. (Note that the usage in a bool context may also block.) """ def __init__(self, cache, title): self.title = title self.cache = cache self._result = AsyncResult() self.edits = {} def _set_result(self, contents, page_info): self._contents = contents self.page_info = page_info self._result.set() @property def contents(self): """Return true if the page exists on the wiki""" self._result.get() return self._contents @property def exists(self): """Return true if the page exists on the wiki""" return self.contents is not None @property def text(self): """Return the contents of the page; raise ValueError if page missing""" self._result.get() if self.exists: return self.contents else: raise ValueError('Page does not exist') def __bool__(self): return self.exists __nonzero__ = __bool__ def edit(self, text, section=None): self._result.get() if not self.page_info['edittoken']: raise ValueError('This Page is not editable') else: self.edits[section] = text EditRequest(self.cache, self).go()
class LogListenerTask(Task): def __init__(self, listener_name, filter_, callback, contract_translator, events_poll_timeout=DEFAULT_EVENTS_POLL_TIMEOUT): super(LogListenerTask, self).__init__() self.listener_name = listener_name self.filter_ = filter_ self.callback = callback self.contract_translator = contract_translator self.stop_event = AsyncResult() self.sleep_time = events_poll_timeout # exposes the AsyncResult timer, this allows us to raise the timeout # inside this Task to force an update: # # task.kill(task.timeout) # self.timeout = None def __repr__(self): return '<LogListenerTask {}>'.format(self.listener_name) def _run(self): # pylint: disable=method-hidden stop = None while stop is None: filter_changes = self.filter_.changes() for log_event in filter_changes: log.debug('New Events', task=self.listener_name) event = self.contract_translator.decode_event( log_event['topics'], log_event['data'], ) if event is not None: originating_contract = log_event['address'] try: self.callback(originating_contract, event) except: log.exception('unexpected exception on log listener') self.timeout = Timeout(self.sleep_time) # wait() will call cancel() stop = self.stop_event.wait(self.timeout) def stop(self): self.stop_event.set(True)
class ReusableCursor(Greenlet): def __init__(self, pool, key, sql, values): super(ReusableCursor, self).__init__(self.work) self.pool = pool self.key = self._formatted_info = key self.sql = sql self.values = values self.offset = 0 self.queue = Queue() self._count = AsyncResult() self.last_access = time.time() self.idle = False self.listeners = [] self.window = config.xgetint('web', 'query-reusable-window', 30) @property def count(self): return self._count.get() def work(self): try: with self.pool.connection() as conn: cur = conn.cursor('_cur') cur.execute(self.sql, self.values) logging.debug(cur.query) cur_tmp = conn.cursor() cur_tmp.execute('MOVE ALL FROM _cur') self._count.set(int(cur_tmp.statusmessage.split()[-1])) cur_tmp.close() cur.scroll(0, 'absolute') while True: if not self.queue.qsize(): self.idle = True for l in self.listeners: spawn(l.onIdle, self) result, limit, offset = self.queue.get(timeout=self.window) self.idle = False if limit is None: raise Killed(result) if self.offset != offset: cur.scroll(offset, 'absolute') data = cur.fetchmany(limit) self.offset = offset + limit result.set(data) self.last_access = time.time() except Empty: pass except Killed, k: k.result.set() finally:
class Future: def __init__(self): self.result = AsyncResult() def set(self, value): self.result.set(value) def get(self): return self.result.get() def on_ready(self, func): while self.result.ready() != False: gevent.sleep(0) func(self.result.get())
class Task(object): def __init__(self, _id, func, args, kwargs): self.id = _id self.result = AsyncResult() self.func = func self.args = args self.kwargs = kwargs def execute(self): try: self.result.set(self.func(*self.args, **self.kwargs)) except Exception, e: log.exception('Task.execute: execution failed.') self.result.set_exception(e)
class AnswerListener(Listener): next_result = None def on_message(self, message): if isinstance(message.result, OperationResult): self.handle_message(message.result) def expect_message(self): if self.next_result: raise ApeException('already waiting for a result!') self.next_result = AsyncResult() return self.next_result def handle_message(self, result): if self.next_result: self.next_result.set(result) self.next_result = None
def handshake(self): """returns None on error or (node_host, node_port) """ data = {} data["name"] = base64.standard_b64encode(login.encrypt('frontend', socket.gethostname())) data["system"] = platform.system() data["machine"] = platform.machine() data["platform"] = platform.platform() data["release"] = platform.release() data["version"] = platform.version() result = AsyncResult() rid = str(id(result)) self.login_results[rid] = result key = login.generate_backend_key() from .. import patch payload = { 'id': rid, 'version': proto.VERSION, 'branch': patch.config.branch, 'commit_id': patch.core_source.version, 'l': login.get('login'), 'system': data, } message_key = proto.pack_message('backend', 'api.set_key', payload=dict(key=key), encrypt="rsa") message = proto.pack_message('backend', 'api.login', payload=payload) try: self.send_message(message_key) self.send_message(message) except AttributeError: result.set([False, 'Client login error']) return try: result = result.get(timeout=20) except gevent.Timeout: result = ["False", "Login timed out"] finally: try: del self.login_results[rid] except KeyError: pass if not result[0]: log.error('login failed: {}'.format(result[1])) if result[1] == 'Invalid Login Credentials': self.connect_retry = 0 login.logout() return False return True
class Task(object): def __init__(self, _id, func, args, kwargs): self.id = _id self.result = AsyncResult() self.func = func self.args = args self.kwargs = kwargs def execute(self): try: self.result.set(self.func(*self.args, **self.kwargs)) except Exception, e: log.exception('Task.execute: execution failed.') self.result.set_exception(e)
class InstrumentAgentEventSubscribers(object): """ Create subscribers for agent and driver events. """ log.info("Start event subscribers") def __init__(self, instrument_agent_resource_id=None): # Start event subscribers, add stop to cleanup. self.no_events = None self.events_received = [] self.async_event_result = AsyncResult() self.event_subscribers = [] def consume_event(*args, **kwargs): log.debug( "#**#**# Event subscriber (consume_event) recieved ION event: args=%s, kwargs=%s, event=%s.", str(args), str(kwargs), str(args[0]), ) log.debug("self.no_events = " + str(self.no_events)) log.debug("self.event_received = " + str(self.events_received)) self.events_received.append(args[0]) if self.no_events and self.no_events == len(self.events_received): log.debug("CALLING self.async_event_result.set()") self.async_event_result.set() self.event_subscribers = EventSubscriber( event_type="ResourceAgentEvent", callback=consume_event, origin=instrument_agent_resource_id ) self.event_subscribers.start() self.event_subscribers._ready_event.wait(timeout=5) def clear_events(self): """ Reset event counter """ self._events_received = [] def stop(self): try: self.event_subscribers.stop() except Exception as ex: log.warn("Failed to stop event subscriber gracefully (%s)" % ex) self.event_subscribers = []
class ClientMessage(object): """RPC client message that is waiting for a result""" queue = None def __init__(self, client): self.client = client self.uuid = uuid4().bytes self.last = time.time() self.result = AsyncResult() def init_stream(self): if self.queue is None: self.result.set(None) self.queue = Queue()
class _SyncCall(object): def __init__(self, name, *args, **kwargs): self.name = name self._args = args self._kwargs = kwargs self._result = AsyncResult() def wait(self, timeout): return self._result.get(timeout=timeout) def execute(self, target): try: function = getattr(target, self.name) self._result.set(function(*self._args, **self._kwargs)) except Exception as error: self._result.set_exception(error)
class RunnableTest(Runnable): def __init__(self): super().__init__() def start(self): self._stop_event = AsyncResult() super().start() def _run(self, *args: Any, **kwargs: Any) -> None: while self._stop_event and self._stop_event.wait(0.5) is not True: gevent.sleep(0.1) return def stop(self): if self._stop_event: self._stop_event.set(True)
def test_heartbeat_failure(self): self.patch_cfg('pyon.ion.process.CFG', {'cc':{'timeout':{'heartbeat_proc_count_threshold':2, 'heartbeat':1.0}}}) svc = self.container.proc_manager.procs[self.pid] ip = svc._process stopar = AsyncResult() self.container.proc_manager.add_proc_state_changed_callback(lambda *args: stopar.set(args)) noticear = AsyncResult() # notify us when the call has been made ar = ip._routing_call(svc.takes_too_long, None, noticear=noticear) noticear.get(timeout=10) # wait for the call to be made # heartbeat a few times so we trigger the failure soon for x in xrange(2): ip.heartbeat() # wait for ip thread to kick over ip._ctrl_thread.join(timeout=5) # now wait for notice proc got canned stopargs = stopar.get(timeout=5) self.assertEquals(stopargs, (svc, ProcessStateEnum.FAILED, self.container)) # should've shut down, no longer in container's process list self.assertEquals(len(self.container.proc_manager.procs), 0)
class FutureResult(object): """ Future results for asynchronous operations. """ def __init__(self): self._result = AsyncResult() self.created_at = time.time() def get(self, timeout=None): return self._result.get(block=True, timeout=timeout) def set(self, value): self._result.set(value) def set_exception(self, exception): self._result.set_exception(exception)
def _sync_call(self, func, cb_arg, *args, **kwargs): """ Functionally similar to the generic blocking_cb but with error support that's Channel specific. """ ar = AsyncResult() def cb(*args, **kwargs): ret = list(args) if len(kwargs): ret.append(kwargs) ar.set(ret) eb = lambda ch, *args: ar.set(TransportError("_sync_call could not complete due to an error (%s)" % args)) kwargs[cb_arg] = cb with self._push_close_cb(eb): func(*args, **kwargs) ret_vals = ar.get(timeout=10) if isinstance(ret_vals, TransportError): # mark this channel as poison, do not use again! # don't test for type here, we don't want to have to import PyonSelectConnection if hasattr(self._client.transport, 'connection') and hasattr(self._client.transport.connection, 'mark_bad_channel'): self._client.transport.connection.mark_bad_channel(self._client.channel_number) else: log.warn("Could not mark channel # (%s) as bad, Pika could be corrupt", self._client.channel_number) raise ret_vals if len(ret_vals) == 0: return None elif len(ret_vals) == 1: return ret_vals[0] return tuple(ret_vals)
def test__control_flow_expired_call(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) self.addCleanup(p.stop) def make_call(call, ctx, val): ar = p._routing_call(call, ctx, val) return ar.get(timeout=10) ctx = { 'reply-by' : 0 } # no need for real time, as it compares by CURRENT >= this value futurear = AsyncResult() with patch('pyon.ion.process.greenlet') as gcm: waitar = AsyncResult() gcm.getcurrent().kill.side_effect = lambda *a, **k: waitar.set() ar = p._routing_call(futurear.set, ctx, sentinel.val) waitar.get(timeout=10) # futurear is not set self.assertFalse(futurear.ready()) # neither is the ar we got back from routing_call self.assertFalse(ar.ready()) # we should've been killed, though self.assertEquals(gcm.getcurrent().kill.call_count, 1) self.assertIsInstance(gcm.getcurrent().kill.call_args[1]['exception'], IonTimeout) # put a new call through (to show unblocked) futurear2 = AsyncResult() ar2 = p._routing_call(futurear2.set, MagicMock(), sentinel.val2) ar2.get(timeout=2)
class InstrumentAgentEventSubscribers(object): """ Create subscribers for agent and driver events. """ log.info("Start event subscribers") def __init__(self, instrument_agent_resource_id=None): # Start event subscribers, add stop to cleanup. self.no_events = None self.events_received = [] self.async_event_result = AsyncResult() self.event_subscribers = [] def consume_event(*args, **kwargs): log.debug( '#**#**# Event subscriber (consume_event) recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) log.debug("self.no_events = " + str(self.no_events)) log.debug("self.event_received = " + str(self.events_received)) self.events_received.append(args[0]) if self.no_events and self.no_events == len(self.events_received): log.debug("CALLING self.async_event_result.set()") self.async_event_result.set() self.event_subscribers = EventSubscriber( event_type='ResourceAgentEvent', callback=consume_event, origin=instrument_agent_resource_id) self.event_subscribers.start() self.event_subscribers._ready_event.wait(timeout=5) def clear_events(self): """ Reset event counter """ self.events_received = [] def stop(self): try: self.event_subscribers.stop() except Exception as ex: log.warn("Failed to stop event subscriber gracefully (%s)" % ex) self.event_subscribers = []
class G(object): def __init__(self, pool_size=300, timeout=1): self.pool_size = pool_size self.timeout = timeout logging.basicConfig(format='%(message)s') self.logger = logging.getLogger(name='g') self.logger.setLevel(logging.INFO) self.pool = Pool(self.pool_size) self.async_result = AsyncResult() def url(self, ip, params=None): if params: suffix = '/search?q=%s' % urllib.quote_plus(' '.join(params)) else: suffix = '' return 'http://%s%s' % (ip, suffix) def worker(self, ip): if not self.async_result.ready(): try: resp = requests.head(self.url(ip), timeout=self.timeout) except requests.exceptions.RequestException: pass # ignore timeouts, connection errors else: if resp.status_code == requests.codes.ok: if not self.async_result.ready(): self.async_result.set(ip) def run(self): # read and parse ip json file gdir = os.path.dirname(__file__) path = os.path.join(gdir, 'ips.json') ips = json.load(open(path)) # start a thread for pool gevent.spawn(self.pool.map, self.worker, ips) # block main thread for the 1st available IP try: ip = self.async_result.get(timeout=5) except gevent.timeout.Timeout: self.logger.info('Timeout (5s).') else: self.pool.kill() self.logger.info(ip) webbrowser.open(self.url(ip, params=sys.argv[1:]))
def taker_commit_async(self, offer): # type: (OfferDeprecated) -> AsyncResult result = AsyncResult() success = self._commitment_service_global.try_take_offer(self, offer.offer_id) if success is False: result.set(None) return result offer_msg = self.create_offer_msg(offer) commitment_msg = messages.TakerCommitment(offer.offer_id, offer_msg.hash, offer.timeout_date, 42) self._sign(commitment_msg) commitment_proof_msg = messages.CommitmentProof(commitment_msg.signature) self._global_sign(commitment_proof_msg) proven_commitment_msg = messages.ProvenCommitment(commitment_msg, commitment_proof_msg) self._sign(proven_commitment_msg) result.set(proven_commitment_msg) self.message_broker.broadcast(self.create_taken(offer.offer_id)) return result
class KorbitGetOrderBookTask(object): def __init__(self, injector, target): self.client = injector.korbit_client self.target = target self.async_result = AsyncResult() def execute(self): gevent.spawn(self.get_orderbook) return self.async_result def get_orderbook(self): try: _, equity_pair = self.target.orderbook_key.split(':') orderbook = self.client.get_orderbook(equity_pair=equity_pair) self.async_result.set(orderbook) except Exception as e: self.async_result.set_exception(e)
def get_dirty_values_async_result(self): return_now = False if self.mode == 'r': log.warn('PersistenceLayer not open for writing: mode=%s', self.mode) return_now = True if self.brick_dispatcher is None: log.debug('\'brick_dispatcher\' is None') return_now = True if return_now: from gevent.event import AsyncResult ret = AsyncResult() ret.set(True) return ret return self.brick_dispatcher.get_dirty_values_async_result()
def get_dirty_values_async_result(self): return_now = False if self.mode == 'r': log.warn('PersistenceLayer not open for writing: mode=%s', self.mode) return_now = True if self.brick_dispatcher is None: log.debug('\'brick_dispatcher\' is None') return_now = True if return_now: from gevent.event import AsyncResult ret = AsyncResult() ret.set(True) return ret return self.brick_dispatcher.get_dirty_values_async_result()
def register_secret_batch(self, secrets: List[typing.Secret]): secrets_to_register = list() secrethashes_to_register = list() secrethashes_not_sent = list() secret_registry_transaction = AsyncResult() for secret in secrets: secrethash = sha3(secret) secrethash_hex = encode_hex(secrethash) is_register_needed = (not self.check_registered(secrethash) and secret not in self.open_secret_transactions) if is_register_needed: secrets_to_register.append(secret) secrethashes_to_register.append(secrethash_hex) self.open_secret_transactions[ secret] = secret_registry_transaction else: secrethashes_not_sent.append(secrethash_hex) log_details = { 'node': pex(self.node_address), 'contract': pex(self.address), 'secrethashes': secrethashes_to_register, 'secrethashes_not_sent': secrethashes_not_sent, } if not secrets_to_register: log.debug('registerSecretBatch skipped', **log_details) return log.debug('registerSecretBatch called', **log_details) try: transaction_hash = self._register_secret_batch(secrets_to_register) except Exception as e: log.critical('registerSecretBatch failed', **log_details) secret_registry_transaction.set_exception(e) raise else: log.info('registerSecretBatch successful', **log_details) secret_registry_transaction.set(transaction_hash) finally: for secret in secrets_to_register: self.open_secret_transactions.pop(secret, None)
class AsyncResultCollector(object): def __init__(self, callNum): self.log = logging.getLogger("{module}.{name}".format( module=self.__class__.__module__, name=self.__class__.__name__)) self.callNum = callNum self.results = {} self.ready = False self.exception = None self.asyncResult = AsyncResult() def return_response(self): if self.exception: raise self.exception if len(list(self.results.values())) > 1: return self.results else: key, value = self.results.popitem() return value def get(self, block=True, timeout=None): if len(list(self.results.values())) == self.callNum: return self.return_response() try: self.log.debug("Waiting for result in blocking call") self.asyncResult.get(timeout=timeout) except gevent.timeout.Timeout as e: return None return self.return_response() def set_exception(self, node, e): self.exception = e self.results[node] = e if len(list(self.results.values())) == self.callNum: self.ready = True self.asyncResult.set() def set(self, node, msg): self.results[node] = msg if len(list(self.results.values())) == self.callNum: self.ready = True self.asyncResult.set()
class AsyncResultCollector(object): def __init__(self, callNum): self.log = logging.getLogger("{module}.{name}".format( module=self.__class__.__module__, name=self.__class__.__name__)) self.callNum = callNum self.results = {} self.ready = False self.exception = None self.asyncResult = AsyncResult() def return_response(self): if self.exception: raise self.exception if len(list(self.results.values())) > 1: return self.results else: key, value = self.results.popitem() return value def get(self, block=True, timeout=None): if len(list(self.results.values())) == self.callNum: return self.return_response() try: self.log.debug("Waiting for result in blocking call") self.asyncResult.get(timeout=timeout) except gevent.timeout.Timeout as e: return None return self.return_response() def set_exception(self, node, e): self.exception = e self.results[node] = e if len(list(self.results.values())) == self.callNum: self.ready = True self.asyncResult.set() def set(self, node, msg): self.results[node] = msg if len(list(self.results.values())) == self.callNum: self.ready = True self.asyncResult.set()
def run(self): self.init_state() self.prompt() if self.input is None: self.p_gen = tokizer(self._do_parse,self.job,self.stop_client) return syn = AsyncResult() self.start_job("job",self._run,syn) self.p_gen = tokizer(self._do_parse,self.job) syn.set(None) try: e = self.job.get() if isinstance(e,BaseException): reraise(e) except StopParsing: pass self.p_gen.exit()
def run(self): self.init_state() self.prompt() if self.input is None: self.p_gen = tokizer(self._do_parse, self.job, self.stop_client) return syn = AsyncResult() self.start_job("job", self._run, syn) self.p_gen = tokizer(self._do_parse, self.job) syn.set(None) try: e = self.job.get() if isinstance(e, BaseException): reraise(e) except StopParsing: pass self.p_gen.exit()
def _cancel_consumer(self, channel, consumer_tag): fut = AsyncResult() self._consumer_cancel_fut = fut channel.basic_cancel( consumer_tag=consumer_tag, callback=lambda _: fut.set(True), ) fut.get() self._consumer_cancel_fut = None self._pika_consumers[channel.channel_number].remove(consumer_tag)