def test_async_multi_publish_consume(): with conn.channel() as channel: # first message message_body = 'test_async_multi_publish_consume message 1' channel.basic_publish( exchange='unit_test_room', routing_key='user1', body=message_body ) recv_queue = Queue() rchannel = conn.allocate_channel() rchannel.basic_consume(queue='listener1', callback=recv_queue.put) resp = recv_queue.get() eq_(resp.body, message_body) resp.ack() assert recv_queue.empty() with conn.channel() as channel: # second message message_body = 'test_async_multi_publish_consume message 2' channel.basic_publish( exchange='unit_test_room', routing_key='user1', body=message_body ) resp = recv_queue.get() eq_(resp.body, message_body) resp.ack()
def joinall(greenlets, timeout=None, raise_error=False, count=None): from gevent.queue import Queue queue = Queue() put = queue.put if count is None: count = len(greenlets) timeout = Timeout.start_new(timeout) try: try: for greenlet in greenlets: greenlet.rawlink(put) if raise_error: for _ in xrange(count): greenlet = queue.get() if not greenlet.successful(): raise greenlet.exception else: for _ in xrange(count): queue.get() except: if sys.exc_info()[1] is not timeout: raise finally: for greenlet in greenlets: greenlet.unlink(put) finally: timeout.cancel()
def process_execution(self, temp_vector, qc_params, bad_times): interval_key = uuid4().hex data_product_id = self.make_large_dataset(temp_vector) async_queue = Queue() def cb(event, *args, **kwargs): if '_'.join(event.qc_parameter.split('_')[1:]) not in qc_params: # I don't care about return times = event.temporal_values self.assertEquals(len(times), bad_times) async_queue.put(1) es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=data_product_id, callback=cb, auto_delete=True) es.start() self.addCleanup(es.stop) config = DotDict() config.process.interval_key = interval_key config.process.qc_params = qc_params self.sync_launch(config) # So now the process is started, time to throw an event at it ep = EventPublisher(event_type='TimerEvent') ep.publish_event(origin=interval_key) try: async_queue.get(timeout=120) except Empty: raise AssertionError('QC was not flagged in time')
def test_last_update_cache(self): handle = self.start_worker() queue = Queue() o_process = handle.process def new_process(msg): o_process(msg) queue.put(True) handle.process = new_process definition = SBE37_CDM_stream_definition() publisher = Publisher() stream_def_id = self.pubsub_cli.create_stream_definition(container=definition) stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id) time = float(0.0) for granule in self.make_points(definition=definition, stream_id=stream_id, N=10): publisher.publish(granule, to_name=(self.XP, stream_id+'.data')) # Determinism sucks try: queue.get(timeout=5) except Empty: self.assertTrue(False, 'Process never received the message.') doc = self.db.read(stream_id) ntime = doc.variables['time'].value self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.') time = ntime
def process_execution(self, temp_vector, qc_params, bad_times): interval_key = uuid4().hex data_product_id = self.make_large_dataset(temp_vector) async_queue = Queue() def cb(event, *args, **kwargs): times = event.temporal_values self.assertEquals(len(times), bad_times) async_queue.put(1) es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=data_product_id, callback=cb, auto_delete=True) es.start() self.addCleanup(es.stop) config = DotDict() config.process.interval_key = interval_key config.process.qc_params = qc_params self.process_dispatcher.schedule_process(self.process_definition_id, process_id=self.process_id, configuration=config) # The process is now up and running maybe? self.sync_launch(config) # So now the process is started, time to throw an event at it ep = EventPublisher(event_type='TimerEvent') ep.publish_event(origin=interval_key) try: for i in xrange(24): async_queue.get(timeout=10) except Empty: raise AssertionError('QC was not flagged in time: %d', i)
def test_qc_interval_integration(self): # 1 need to make a dataset that only has one discrete qc violation # 2 Launch the process # 3 Setup the scheduler to run it say three times # 4 Get the Events and verify the data #-------------------------------------------------------------------------------- # Make a dataset that has only one discrete qc violation #-------------------------------------------------------------------------------- dp_id, dataset_id, stream_def_id = self.make_data_product() ph = ParameterHelper(self.dataset_management, self.addCleanup) monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) for rdt in self.populate_vectors(stream_def_id, 1, lambda x : [41] + [39] * (x-1)): ph.publish_rdt_to_data_product(dp_id, rdt) self.assertTrue(monitor.event.wait(10)) # Give it 10 seconds to populate #-------------------------------------------------------------------------------- # Launch the process #-------------------------------------------------------------------------------- interval_key = uuid4().hex config = DotDict() config.process.interval_key = interval_key config.process.qc_params = ['glblrng_qc'] # The others are tested in other tests for completeness self.sync_launch(config) async_queue = Queue() def callback(event, *args, **kwargs): times = event.temporal_values self.assertEquals(len(times), 1) async_queue.put(1) es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=dp_id, callback=callback, auto_delete=True) es.start() self.addCleanup(es.stop) #-------------------------------------------------------------------------------- # Setup the scheduler #-------------------------------------------------------------------------------- timer_id = self.scheduler_service.create_interval_timer(start_time=time.time(), end_time=time.time()+13, interval=5, event_origin=interval_key) #-------------------------------------------------------------------------------- # Get the events and verify them #-------------------------------------------------------------------------------- try: for i in xrange(2): async_queue.get(timeout=10) except Empty: raise AssertionError('QC Events not raised')
class TestBase(unittest.TestCase): class MockConfig(object): def __init__(self): self.snmp_host = "127.0.0.1" self.snmp_port = 1337 def setUp(self): self.log_queue = Queue() dom = etree.parse('templates/default.xml') mibs = dom.xpath('//conpot_template/snmp/mibs/*') #only enable snmp server if we have configuration items if not mibs: raise Exception("No configuration for SNMP server") else: self.snmp_server = snmp_command_responder.CommandResponder(self.log_queue, server_config=self.MockConfig()) for mib in mibs: mib_name = mib.attrib['name'] for symbol in mib: symbol_name = symbol.attrib['name'] value = symbol.xpath('./value/text()')[0] self.snmp_server.register(mib_name, symbol_name, value) self.snmp_server.snmpEngine.transportDispatcher.start() def tearDown(self): self.snmp_server.snmpEngine.transportDispatcher.stop() def mock_callback(self, sendRequestHandle, errorIndication, errorStatus, errorIndex, varBindTable, cbCtx): self.result = None if errorIndication: self.result = errorIndication elif errorStatus: self.result = errorStatus.prettyPrint() else: for oid, val in varBindTable: self.result = val.prettyPrint() def test_snmp_get(self): client = snmp_client.SNMPClient(client_config=self.MockConfig()) OID = ((1, 3, 6, 1, 2, 1, 1, 1, 0), None) client.get_command(OID, callback=self.mock_callback) self.assertEqual("Siemens, SIMATIC, S7-200", self.result) log_item = self.log_queue.get(True, 2) self.assertIsInstance(log_item['timestamp'], datetime) self.assertEqual('127.0.0.1', log_item['remote'][0]) self.assertEquals('snmp', log_item['data_type']) def test_snmp_set(self): client = snmp_client.SNMPClient(client_config=self.MockConfig()) OID = ((1, 3, 6, 1, 2, 1, 1, 6, 0), rfc1902.OctetString('test comment')) client.set_command(OID, callback=self.mock_callback) # FIXME: no log entry for set commands client.get_command(OID, callback=self.mock_callback) self.assertEqual("test comment", self.result) get_log_item = self.log_queue.get(True, 5) self.assertIsInstance(get_log_item['timestamp'], datetime) self.assertEqual('127.0.0.1', get_log_item['remote'][0]) self.assertEquals('snmp', get_log_item['data_type'])
class MemorySession(Session): """ In memory session with a outgoing gevent Queue as the message store. """ def __init__(self, server, session_id=None): super(MemorySession, self).__init__(server, session_id=session_id) self.session_id = session_id or str(uuid.uuid4())[:8] self.server = server self.queue = Queue() self.hits = 0 self.heartbeats = 0 self.connected = False def add_message(self, msg): self.queue.put_nowait(msg) def get_messages(self, **kwargs): timeout = kwargs.get('timeout', None) self.incr_hits() if self.queue.empty(): try: return self.queue.get(**kwargs) except Empty: return [] else: accum = [] try: while not self.queue.empty(): if timeout: accum.append(self.queue.get(timeout=timeout)) else: accum.append(self.queue.get_nowait()) finally: return accum def interrupt(self): """ A kill event trigged through a client accessible endpoint Internal expires will not have is_interupted() == True """ self.interrupted = True self.kill() def kill(self): self.connected = False # Expire only once if not self.expired: self.expired = True self.timeout.set()
def test_link_to_queue(self): p = gevent.spawn(lambda: 100) q = Queue() p.link(q.put) self.assertEqual(q.get().get(), 100) for i in range(3): p.link(q.put) self.assertEqual(q.get().get(), 100)
class UA(object): def __init__(self, app, stack): self.app, self.options, self._stack = app, app.options, stack self._closeQ = self._ua = self._gen = None self.scheme = self._stack.transport.secure and "sips" or "sip" self.localParty = rfc2396.Address("%s:%s@%s" % (self.scheme, self.options.user, self.options.domain)) self.proxy = rfc2396.URI("%s:%s" % (self.scheme, self.options.proxy)) if self.options.proxy else None def _waitOnClose(self): # Wait on close event to be signaled by another task if self._closeQ is None: self._closeQ = Queue() self._closeQ.get() self._closeQ = None else: raise ValueError("some other task is already waiting on close") def _signalClose(self): # Signal the close event on this object. if self._closeQ is not None: self._closeQ.put(None) def _createClient(self, localParty, remoteParty, remoteTarget): ua = self._ua = rfc3261.UserAgent(self._stack) ua.app, ua.localParty, ua.remoteParty, ua.remoteTarget = ( self, localParty.dup(), remoteParty.dup(), remoteTarget.dup(), ) def _scheduleRefresh( self, response, handler ): # Schedule handler to be invoked before response.Expires or self._interval interval = int(response.Expires.value) if response.Expires else self._interval interval = max(interval - random.randint(5000, 15000) / 1000.0, 5) if interval > 0: logger.debug("scheduling refresh after %r", interval) self._gen = gevent.spawn_later(interval, handler) def _scheduleRetry(self, handler): # Schedule handler to be invoked after retry_interval. logger.debug("scheduling retry after %r", self.options.retry_interval) self._gen = gevent.spawn_later(self.options.retry_interval, handler) def _closeGen(self): if self._gen is not None: self._gen.kill() self._gen = None def _closeUA(self): if self._ua is not None: self._ua.app = None self._ua = None self.app = None # remove reference def dialogCreated(self, ua, dialog): # Invoked by SIP stack to inform that UserAgent is converted to Dialog. if self._ua is not None: self._ua.app = None self._ua, dialog.app = dialog, self
def test_link_to_channel(self): p1 = gevent.spawn(lambda: 101) p2 = gevent.spawn(lambda: 102) p3 = gevent.spawn(lambda: 103) q = Queue(0) p1.link(q.put) p2.link(q.put) p3.link(q.put) results = [q.get().get(), q.get().get(), q.get().get()] assert sorted(results) == [101, 102, 103], results
def bad(): q = Queue() print('switching out, then throwing in') try: q.get(block=True, timeout=0.1) except Empty: print("Got Empty") print('switching out') gevent.sleep(0.1) print('switched in')
class Rig(object): def __init__(self, device): self.txlines = Queue() self.rxlines = Queue() self.tty = serial.Serial(device, 9600, timeout=1) gevent.spawn(self.receiver) gevent.spawn(self.sender) self.vfos = (VFO(), VFO()) self.con_vfo = None self.tx_vfo = None def receiver(self): rxbuf = [] while True: char = self.tty.read(1) if char == "": continue elif char == "\r": log.debug("read %s" % repr("".join(rxbuf))) self.rxlines.put("".join(rxbuf)) rxbuf = [] elif char != "": rxbuf.append(char) def sendline(self, line): self.txlines.put(line) def sender(self): while True: line = self.txlines.get() log.debug("write %s" % repr(line)) self.tty.write(line + "\r") def print_lines(self): while True: print "AI %s" % repr(self.rxlines.get()) def close(self): self.tty.close() def __enter__(self, *args, **kwargs): pass def __exit__(self, *args, **kwargs): self.close() @property def freq(self): self.tty.write("BUF 0\n") return self.tty.readline() @freq.setter def freq(self, hz): pass
def waitall(events): # QQQ add timeout? from gevent.queue import Queue queue = Queue() put = queue.put try: for event in events: event.rawlink(put) for _ in xrange(len(events)): queue.get() finally: for event in events: event.unlink(put)
def init_tasks(self): filenames = list() for task_str in self.tasks_str: if task_str.isnumeric(): info = self.client.get_info(task_str.encode('utf-8')) obj = FileInfo.create_from_download_info(info, self.filename) if obj.filename not in filenames: filenames.append(obj.filename) else: if task_str not in filenames: filenames.append(task_str) file_infos = list(map(FileInfo.get, filenames)) """:type: List[FileInfo]""" DBSession().expunge_all() gevent.spawn(self.start_server, file_infos) inq = Queue() filename_outq_map = {} """:type: Dict[str, Queue]""" filename_info_map = {} """:type: Dict[str, FileInfo]""" for filename in filenames: filename_outq_map[filename] = Queue() for info in file_infos: assert isinstance(info, FileInfo) assert info.requests, "%s.request is empty" % info.filename filename_info_map[info.filename] = info undownload_filenames = list(filenames) for _ in range(min(self.max_concurrent, len(undownload_filenames))): inq.put((Classification.FILE_FINISHED, )) downloading_files = 0 while undownload_filenames: data = inq.get() assert data[0] == Classification.FILE_FINISHED filename = undownload_filenames.pop(0) info = filename_info_map[filename] """:type: FileInfo""" if info.is_finished(): logger.info("%s is finished", filename) continue g = gevent.pool.Group() obj = FileDownloader(filename_info_map[filename], filename_outq_map[filename], inq, g) g.spawn(obj.start) downloading_files += 1 while downloading_files: data = inq.get() assert data[0] == Classification.FILE_FINISHED downloading_files -= 1 self.exit_event.set()
class IRP(object): '''I/O Request Packet''' complete_tag = object() def __init__(self): from gevent.queue import Queue self.queue = Queue() def complete(self): # from gevent_extension import the_hub self.queue.put(self.complete_tag) # the_hub.interrupt(self.queue.put, self.complete_tag) def wait(self): self.queue.get()
class Channel(gevent.Greenlet): """Baseline for inter-app communication""" def __init__(self, remote, t, c=None): super(Channel, self).__init__() self.remote = remote self.transmit = remote.send self.c = c self.t = t self.inq = Queue() self.wait_for_roundtrip = False if self.c is None: self.wait_for_roundtrip = True self.c = os.urandom(16).encode('hex') def _run(self): self.running = True if self.wait_for_roundtrip: data, body = self.inq.get() self._recv_first(data, body) #explicit, but not required self.wait_for_roundtrip = False gevent.sleep(0) while self.running: data, body = self.inq.get() self._recv(data, body) def _recv_first(self, data, body): log.debug("Channel %s recv:\n%s" % (self.c, data)) self.handle_unknown(data, body) def _recv(self, data, body): log.debug("Channel %s recv:\n%s" % (self.c, data)) self.handle_unknown(data, body) def send(self, data, body=''): data['c'] = self.c self.transmit(data, body) def handle_unknown(self, data, body): if 'err' in data: log.debug('Remote error: %s' % data['err']) return if 'end' in data: return err = '%s currently unimplemented' % self.t resp = {'end': True, 'err': err} #log.debug('To %s: %s' % (remote.id.hash_name, err)) self.send(resp)
class Actor(gevent.Greenlet): def __init__(self): self.running = False self.inbox = Queue() gevent.Greenlet.__init__(self) def received(self, message): """ Define your subclass to handle incoming messages here... """ raise NotImplementedError() def cast(self, message): """ Send a message to the actor. If the actor is busy, the message will be enqueued for later consumption. There is no return value. >>> a = Actor() >>> a.received = lambda msg: msg >>> a.cast("hello") """ self.inbox.put_nowait(message) def _run(self): self.running = True while self.running: message = self.inbox.get() self.received(message)
class Spider: def __init__(self, url='', depth=1, threads=4): self.url = url self.depth = depth self.threads = threads self.tasks = Queue() self.bucket = [] def run(self): self.tasks.put(Task(self.url, self.depth)) threds = [ gevent.spawn(self.worker) for i in range(self.threads) ] gevent.joinall(threds) def worker(self, worker_id=''): while not self.tasks.empty(): task = self.tasks.get() if task.url in self.bucket: # here have a bug continue self.bucket.append(task.url) task.run() for t in task.subtasks: self.tasks.put_nowait(t)
def parallel_loop(func=None, elem=None, *args, **kwargs): import gevent from gevent.queue import Queue ret = [] api.bui.cli.logger.debug('Using gevent') if not callable(func): api.abort(500, 'The provided \'func\' is not callable!') if not elem: return [] output = Queue() processes = [ gevent.spawn( func, e, output, *args, **kwargs ) for e in elem ] # wait for process termination gevent.joinall(processes) for p in processes: tmp = output.get() if isinstance(tmp, basestring): api.abort(500, tmp) elif tmp: ret.append(tmp) return ret
class C2DMService(object): def __init__(self, source, email, password): self.source = source self.email = email self.password = password self._send_queue = Queue() self._send_queue_cleared = Event() self.log = logging.getLogger('pulsus.service.c2dm') def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: self.log.info("C2DM service started") while True: notification = self._send_queue.get() try: self._do_push(notification) except Exception, e: self.log.exception("Error while pushing") self._send_queue.put(notification) gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class BlackBerryPushService(object): def __init__(self, app_id, password, push_url): self.app_id = app_id self.password = password self.push_url = push_url self._send_queue = Queue() self._send_queue_cleared = Event() self.log = logging.getLogger('pulsus.service.bbp') def _send_loop(self): self._send_greenlet = gevent.getcurrent() try: self.log.info("BlackBerry Push service started") while True: notification = self._send_queue.get() try: self._do_push(notification) except Exception, e: print e self._send_queue.put(notification) gevent.sleep(5.0) finally: if self._send_queue.qsize() < 1 and \ not self._send_queue_cleared.is_set(): self._send_queue_cleared.set()
class WebSocketClient(ThreadedClient): def __init__(self, url, protocols=None, version='8'): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) ThreadedClient.__init__(self, url, protocols=protocols, version=version, sock=sock) self._lock = Semaphore() self._th = Greenlet(self._receive) self._messages = Queue() self.extensions = [] def opened(self, protocols, extensions): self.protocols = protocols self.extensions = extensions def received_message(self, m): self._messages.put(copy.deepcopy(m)) def write_to_connection(self, bytes): if not self.client_terminated: return self.sock.sendall(bytes) def closed(self, code, reason=None): self._messages.put(StreamClosed(code, reason)) def receive(self, msg_obj=False): msg = self._messages.get() if isinstance(msg, StreamClosed): return None if msg_obj: return msg else: return msg.data
class NotifyingQueue(Event): """ A queue that follows the wait protocol. """ def __init__(self): super(NotifyingQueue, self).__init__() self._queue = Queue() def put(self, item): """ Add new item to the queue. """ self._queue.put(item) self.set() def empty(self): return self._queue.empty() def get(self, block=True, timeout=None): """ Removes and returns an item from the queue. """ value = self._queue.get(block, timeout) if self._queue.empty(): self.clear() return value def stop(self): """ Request a stop event. """ self.set()
class WebSocketHandler(ThreadedHandler): """WebSocket API for handlers This provides a socket-like interface similar to the browser WebSocket API for managing a WebSocket connection. """ def __init__(self, sock, protocols, extensions, environ): ThreadedHandler.__init__(self, sock, protocols, extensions) self.environ = environ self._messages = Queue() self._lock = Lock() self._th = gevent.spawn(self._receive) def closed(self, code, reason=None): self._messages.put(StreamClosed(code, reason)) def received_message(self, m): self._messages.put(copy.deepcopy(m)) def receive(self, msg_obj=False): msg = self._messages.get() if isinstance(msg, StreamClosed): # Maybe we'll do something better return None if msg_obj: return msg else: return msg.data
class CrawlBase(object): spider_count = 0 def __init__(self): self.group = Group() self.queue = Queue() def read_seed(self, file='seeds.txt'): with open(file) as f: for line in f: if len(line) > 0 and line != "\n": yield line.strip() else: return def dispatch(self): for url in self.read_seed(): g = gevent.spawn(Spider, self, url) self.group.add(g) self.group.join() def harvest(self): try: while True: content = self.queue.get(timeout=2) print(content) except Empty: pass
class CHubCallbackQueueBase(CHubCallbackBasicBase): def __init__(self,sHubId): CHubCallbackBasicBase.__init__(self,sHubId) self.__queue4Return = Queue() #当前应答队列 def PutCmdStrToReturnQueue(self, lsCmdStr): self.__queue4Return.put(lsCmdStr) def GetCmdStrFmReturnQueue(self): return self.__queue4Return.get() # (sClientIPPort,dwCmdId,CmdOStr) def HandleRequestCmd(self, sClientIPPort, dwCmdId, CmdIStr): # 处理客户端请求命令 bDone = CHubCallbackBasicBase.HandleRequestCmd(self, sClientIPPort, dwCmdId, CmdIStr) if not bDone and CmdIStr[0].startswith(CMD0_ECHO_CMD): CmdOStr = ['OK','CHubCallbackQueueBase'] CmdOStr.extend(CmdIStr) dwCmdId = GetCmdReplyFmRequest(dwCmdId) self.PutCmdStrToReturnQueue([sClientIPPort,dwCmdId,CmdOStr]) bDone = True return bDone def DoHandleCheckAllLinkReply(self): # 处理检查所有链接的应答返还消息(包括通知消息)等 # 该函数在该类中实现后,一般情况下子类无需再继承。 while not self.bQuitLoopFlag: return self.GetCmdStrFmReturnQueue()
class FlowTests(TestCase): def create(self, conf={}, events=[]): self.input = Queue() self.output = Queue() context = DummyContext() with context: self.i = self.create_stage(**conf) self.input = self.i.setup(self.output) self.assertEquals(1, len(context.stages)) self.i.start() for ev in events: self.input.put(ev) return self.i def wait(self, timeout=1.0, events=1): with gevent.Timeout(timeout): # wait for input to be consumed and output to be produced while self.input.qsize(): gevent.sleep(0.0) while self.output.qsize() < events: gevent.sleep(0.0) self.i.stop() if events: return [self.output.get() for n in xrange(events)]
class LocalControlProgramDescriptor(object): def __init__(self, hc, agentUuid, program_id): self.log = logging.getLogger('LocalControlProgramDescriptor_{}'.format(program_id)) self.agentUuid = agentUuid self.id = program_id self.hc = hc self.queue = Queue() def _receive_msg(self, msg): self.queue.put(msg) def recv(self, block=True, timeout=None): try: self.log.debug("Waiting for msg in blocking call") msg = self.queue.get(block=block, timeout=timeout) return msg except gevent.timeout.Timeout as e: return None except gevent.queue.Empty as e: return None def send(self, msg): return self.hc.send_msg_to_local_control_program(self.id, msg, self.agentUuid) def close(self): return self.hc.stop_local_control_program(self.id, self.agentUuid)
class Spider: def __init__(self, url='', depth=1): self.tasks = Queue() self.tasks.put(url) self.init_url = url or '' self.depth = depth or '' def run(self): threds = [ gevent.spawn(self.work), gevent.spawn(self.work), gevent.spawn(self.work), gevent.spawn(self.work) ] gevent.joinall(threds) def work(self): while not self.tasks.empty(): page = self.tasks.get() p = Page(page, '') p.do_request() p.parse_content() hrefs = p.hrefs for href in hrefs: self.tasks.put_nowait(href)
def lookup(self, target_key): target_id = keccak256(target_key) closest = [] while not closest: closest = self.closest(target_id, BUCKET_SIZE) if not closest: # add seed nodes for bn in self.server.boot_nodes: self.add_node(bn) asked = [self.self_node.node_id] pending_queries = 0 reply_queue = Queue() while True: for n in closest: if pending_queries >= KAD_ALPHA: break if n.node_id not in asked: asked.append(n.node_id) pending_queries += 1 gevent.spawn(self.find_neighbours, n, target_key, reply_queue) if pending_queries == 0: break ns = reply_queue.get() pending_queries -= 1 if ns: for node in ns: farther = find_farther_to_target_than( closest, target_id, node) if farther: closest.remove(farther) if len(closest) < BUCKET_SIZE: closest.append(node)
class OutputService(BaseService): """ This is a service which one expects output to be tracked. The mechanism is via the use of an output_queue of type gevent.Queue. As opposed to a scheduled service which is almost a fire and forget. Note that a fire and forget is great where it can still log output to an external system for deferred parsing. """ def __init__(self, name, size=None, items=None): BaseService.__init__(self, name) self._queue = Queue(size, items) def put(self, item, block=True, timeout=None): return self._queue.put(item, block, timeout) def get(self, block=True, timeout=None): """ Gets from the backing Queue. If calling get on empty Queue, it will block until Queue actually has contents to return. If that is undesired set block=False to prevent greenlet blocking. :param block: :param timeout: :return: """ return self._queue.get(block, timeout) def size(self): return self.queue_ref().qsize() def queue_ref(self): """ Direct access if necessary. Warning this will allow direct access for modification. Tend not to use it. :return: """ return self._queue
class GeventActor(Greenlet): def __init__(self): super(GeventActor, self).__init__() self._inbox = Queue() self._running = False def tell(self, message): self._inbox.put(message) def on_receive(self, message): raise NotImplementedError() def _run(self): self._running = True while self._running: message = self._inbox.get() if not isinstance(message, Dokka.Messages.Message): continue if isinstance(message, Dokka.Messages.StopActor): self._running = False self.on_receive(message)
class NotifyingQueue(Event): """ A queue that follows the wait protocol. """ def __init__(self): super(NotifyingQueue, self).__init__() self._queue = Queue() def put(self, item): """ Add new item to the queue. """ self._queue.put(item) self.set() def get(self, block=True, timeout=None): """ Removes and returns an item from the queue. """ value = self._queue.get(block, timeout) if self._queue.empty(): self.clear() return value def stop(self): """ Request a stop event. """ self.set()
class BufferedOpusEncoderPlayable(BasePlayable, AbstractOpus, OpusEncoder): def __init__(self, source, *args, **kwargs): self.source = source self.frames = Queue(kwargs.pop('queue_size', 4096)) super(BufferedOpusEncoderPlayable, self).__init__(*args, **kwargs) gevent.spawn(self._encoder_loop) def _encoder_loop(self): while self.source: raw = self.source.read(self.frame_size) if len(raw) < self.frame_size: break self.frames.put(self.encode(raw, self.samples_per_frame)) gevent.idle() self.source = None def next_frame(self): if not self.source: return None return self.frames.get()
def get_from_pool(pool_id): # check and initialize if (boto_sessions[0] == None and config.aws_config): boto_sessions[0] = boto3.session.Session(**config.aws_config) conn = None conn_pool = conn_pools.get(pool_id, None) if (conn_pool == None): # if no conn_pool for given pool_id create and store in conn_pools conn_pool = Queue() conn_pools[pool_id] = conn_pool try: conn = conn_pool.get(block=False) # print "reusing from pool" except Empty: #if queue is empty create a connection for use. if (pool_id in _pool_item_generators): conn = _pool_item_generators[pool_id]() return conn
class NotifyingQueue(Event, Generic[T]): """This is not the same as a JoinableQueue. Here, instead of waiting for all the work to be processed, the wait is for work to be available. """ def __init__(self, maxsize: int = None, items: Iterable[T] = ()) -> None: super().__init__() self.queue = Queue(maxsize, items) if items: self.set() def put(self, item: T) -> None: """ Add new item to the queue. """ self.queue.put(item) self.set() def get(self, block: bool = True, timeout: float = None) -> T: """ Removes and returns an item from the queue. """ value = self.queue.get(block, timeout) if self.queue.empty(): self.clear() return value def peek(self, block: bool = True, timeout: float = None) -> T: return self.queue.peek(block, timeout) def __len__(self) -> int: return len(self.queue) def copy(self) -> List[T]: """ Copies the current queue items. """ copy = self.queue.copy() result = list() while not copy.empty(): result.append(copy.get_nowait()) return result def __repr__(self) -> str: return f"NotifyingQueue(id={id(self)}, num_items={len(self.queue)})"
class FirstReply: def __init__(self): self.size = 0 self.index = 0 self.q = Queue() def push(self, *args, **kwargs): self.size += 1 callback = args[0] args = args[1:] if callback: def handle(): if args and not kwargs: self.q.put(callback(*args)) elif not args and kwargs: self.q.put(callback()) else: self.q.put(callback(*args, **kwargs)) spawn(handle) return self.q def __len__(self): return self.size def __iter__(self): return self def next(self): v = None if self.index < self.size: v = self.q.get() self.index += 1 else: raise StopIteration return v __add__ = push append = push
class GreenletExecutor(AbstractExecutor): """ GreenletExecutor is an AbstractExecutor subclass that uses a pool of greenlets to execute calls asynchronously. NOTE: Use this executor for I/O-bound tasks. Since all greenlets are multiplexed on a single pthread, do NOT use this for compute-bound callables. Try using the GIPCExecutor instead. """ def __init__(self, num_greenlets=50, **kwargs): super(GreenletExecutor, self).__init__(**kwargs) self.pool = Pool(size=num_greenlets) self.task_queue = Queue() self.num_ready = 0 def _shutdown(self): for _ in xrange(len(self.pool)): self.task_queue.put(None) if self.force_kill_on_shutdown: self.pool.kill() else: self.pool.join() def _worker_loop(self): try: self.num_ready += 1 while True: self.num_ready -= 1 task = self.task_queue.get() if task is None: return task.execute() self.num_ready += 1 except: pass def _submit(self, task): self.task_queue.put(task) if not self.num_ready and self.pool.free_count(): self.pool.spawn(self._worker_loop)
class GeventQueue(object): def __init__(self, csvdicts, goal_path, gevent_num=100): # self.function = function self.csvdicts = csvdicts self.goal_path = goal_path self.gevent_num = gevent_num self.tasks = Queue() def worker(self, i): print('worker %d start' % (i)) while not self.tasks.empty(): try: # print('worker %d start'%(i)) task = self.tasks.get() # write current task into worker's running log f = open(os.path.join('logfile', '%s_running.log' % (i)), 'wb') f.write(task) f.close() # run the task Reorganizer(task, self.goal_path).start() # write finished work root, base = os.path.split(task) fs = open(os.path.join(root, 'success'), 'ab') fs.write(base) fs.write('\n') fs.close() gevent.sleep(0) except Exception, e: print(e) # write error work into error log print('error:%d; task:%s' % (i, task)) fl = open(os.path.join('logfile', 'error.log'), 'ab') fl.write(task) fl.write('\n') fl.close() continue
class GeventQueue(object): def __init__(self, old_path, new_path, gevent_num=100): self.old_path = old_path self.new_path = new_path self.gevent_num = gevent_num self.tasks = Queue() def worker(self, i): print('worker %d start' % (i)) while not self.tasks.empty(): try: # print('worker %d start'%(i)) task = self.tasks.get() # write current task into worker's running log f = open( os.path.join('logfile_extractor', '%s_running.log' % (i)), 'wb') f.write(task.encode('utf-8')) f.close() # run the task Extractor(task, self.old_path, self.new_path).start() # write finished work fs = open(os.path.join('extracted'), 'ab') fs.write(task.encode('utf-8')) fs.write('\n') fs.close() gevent.sleep(0) except Exception, e: print(e) # write error work into error log print('error:%d; task:%s' % (i, task)) fl = open(os.path.join('logfile', 'error.log'), 'ab') fl.write(task.encode('utf-8')) fl.write('\n') fl.close() continue
class MyCrawler(object): def __init__(self): self.pool = Pool(10) self.tasks = Queue() def init_tasks(self): resp = requests.get('http://www.ttmeiju.com/summary.html') assert resp.status_code==200 soup = BeautifulSoup(resp.text) tb = soup.find('table') def mya(tag): return tag.name=='a' and tag.parent.name=='td' meiju={} for atag in tb.find_all(mya)[3:]: meiju[atag.string] = atag['href'] self.tasks.put_nowait({'key':atag.string, 'val': atag['href']}) print (meiju) def do_jobs(self): self.init_tasks() while not self.tasks.empty(): task = self.tasks.get() self.before(task) self.pool.spawn(self.work, task).rawlink(self.after) # 等待任务完成 print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') self.pool.join() def work(self, task): print ('子类实现') def before(self, task): print('') def after(self, asynret): print('')
def messages(self): """Listen to the messages WebSocket stream Retrieve the WebSocket stream and send to it each messages displayed by the display method. They are JSON encoded as a dict { style: message }. """ profile = request.params.get("profile", self.actionsmap.default_authentication) authenticator = self.actionsmap.get_authenticator(profile) s_id = authenticator.get_session_cookie()["id"] try: queue = self.log_queues[s_id] except KeyError: # Create a new queue for the session queue = Queue() self.log_queues[s_id] = queue wsock = request.environ.get("wsgi.websocket") if not wsock: raise HTTPResponse(m18n.g("websocket_request_expected"), 500) while True: item = queue.get() try: # Retrieve the message style, message = item except TypeError: if item == StopIteration: # Delete the current queue and break del self.log_queues[s_id] break logger.exception("invalid item in the messages queue: %r", item) else: try: # Send the message wsock.send(json_encode({style: message})) except WebSocketError: break sleep(0)
class MemorySession(session.Session): """ In memory session with a ``gevent.pool.Queue`` as the message store. """ def __init__(self, *args, **kwargs): super(MemorySession, self).__init__(*args, **kwargs) self._queue = Queue() def add_message(self, frame, data): log.info('session closed: %s', self.id) self._queue.put_nowait((frame, data)) #waiter = self._waiter #if waiter is not None: # self._waiter = None # if not waiter.cancelled(): # waiter.set_result(True) self._tick() def get_messages(self, timeout=None): self._tick() messages = [] # get all messages immediately pending in the queue while not self._queue.empty(): try: msg = self._queue.get_nowait() except Empty: break messages.append(msg) if not messages: try: messages.append(self._queue.get(timeout=timeout)) except Empty: pass return messages
def _concurrent_execute(self, context, start_req, parser, pool, pool_size): queue = Queue() # 任务队列 # 将初始化请求加入任务队列 for r in start_req: queue.put_nowait(r) if pool is None: pool = GeventPool(pool_size) greenlets = [] while True: try: req = self._check_req(queue.get(timeout=1)) if req.parser is None: req.parser = parser greenlets.append(pool.spawn(req, context, queue)) except Empty: break return [greenlet.get() for greenlet in greenlets]
class Actor: def __init__(self): self.inbox = Queue() self.greenlet = spawn(self._listen) def _listen(self): try: while True: message = self.inbox.get() self._receive(message) sleep(0) except ActorStop: pass def _receive(self, message): raise NotImplemented() def stop(self): killall([self.greenlet], exception=ActorStop) def post(self, message, block=True, timeout=None): self.inbox.put(message, block, timeout)
def get_from_pool(pool_id='dynamodb'): conn = None conn_pool = conn_pools.get(pool_id, None) if (conn_pool == None): #if no conn_pool for given pool_id create and store in co]nn_pools conn_pool = Queue() conn_pools[pool_id] = conn_pool try: conn = conn_pool.get(block=False) # print "reusing from pool" except Empty: #if queue is empty create a connection for use. if (pool_id.startswith("dynamodb")): #table_name = pool_id.split("_")[1] #create new session if (IS_LOCAL_ENDPOINT[0]): conn = boto3.resource( 'dynamodb', endpoint_url='http://{addr}:{port}'.format( addr=IS_LOCAL_ENDPOINT[0][0], port=IS_LOCAL_ENDPOINT[0][1]), aws_access_key_id=" ", aws_secret_access_key=" ", region_name="ap-south-1") else: conn = boto_session.resource('dynamodb') elif (pool_id.startswith("s3")): conn = boto_session.resource('s3') # print "new connection" elif (pool_id.startswith("sqs")): conn = boto_session.client('sqs') elif (pool_id.startswith("ses")): conn = boto_session.client('ses', region_name="eu-west-1") return conn
class Stream(Expression): def __init__(self, io: bool): self.io = io self.typ = self.Type.STREAM self.q = Queue() self.closed = False self.value = None def is_in(self): return self.io def read(self): if self.closed: return None e = self.q.get() if e == EOS: self.closed = True return None return e def write(self, e): return self.q.put(e)
def send_proc(): q = Queue() if ptoken: if ptoken in tokenSubscriptions: tokenSubscriptions[ptoken].append(q) else: tokenSubscriptions[ptoken] = [q] if not hashstr in subscriptions: subscriptions[hashstr] = [q] else: subscriptions[hashstr].append(q) try: while True: yield "data: %s\n\n" % q.get() # TODO: this may actually not work, need an explicit cleanup here finally: # print "Finally! for ptoken", ptoken, "hash:", hashstr if hashstr in subscriptions: try: subscriptions[hashstr].remove(q) except ValueError: pass if len(subscriptions[hashstr]) == 0: # print "Subs zero", ptoken, "hash:", hashstr try: del subscriptions[hashstr] except KeyError: pass if ptoken and ptoken in tokenSubscriptions: try: tokenSubscriptions[ptoken].remove(q) except ValueError: pass if len(tokenSubscriptions[ptoken]) == 0: # print "Tokens zero!", ptoken, "hash:", hashstr try: del tokenSubscriptions[ptoken] except KeyError: pass
class NotifyingQueue(Event): def __init__(self): super(NotifyingQueue, self).__init__() self._queue = Queue() def put(self, item): """ Add new item to the queue. """ self._queue.put(item) self.set() def get(self, block=True, timeout=None): """ Removes and returns an item from the queue. """ value = self._queue.get(block, timeout) if self._queue.empty(): self.clear() return value def peek(self, block=True, timeout=None): return self._queue.peek(block, timeout) def __len__(self): return len(self._queue)
class Buffer(Greenlet): latency = 0.01 def __init__(self, server, num_frames=1000): self.server = server self.num_frames = num_frames self.frames = Queue(100) Greenlet.__init__(self) def _run(self): for i in range(self.num_frames): # request data # print('sleep') gevent.sleep(self.latency) # print('get') f = self.server.get_frame(i) # print('pre put') self.frames.put(f) # blocks if full # print('post put') def get_frame(self): return self.frames.get() # blocks if empty
def messages(self): """Listen to the messages WebSocket stream Retrieve the WebSocket stream and send to it each messages displayed by the core.MoulinetteSignals.display signal. They are JSON encoded as a dict { style: message }. """ s_id = request.get_cookie('session.id') try: queue = self.log_queues[s_id] except KeyError: # Create a new queue for the session queue = Queue() self.log_queues[s_id] = queue wsock = request.environ.get('wsgi.websocket') if not wsock: raise HTTPErrorResponse(m18n.g('websocket_request_expected')) while True: item = queue.get() try: # Retrieve the message style, message = item except TypeError: if item == StopIteration: # Delete the current queue and break del self.log_queues[s_id] break logger.exception("invalid item in the messages queue: %r", item) else: try: # Send the message wsock.send(json_encode({style: message})) except WebSocketError: break sleep(0)
def command(self, method: str, **params) -> dict: self._seq_no += 1 seq_no = self._seq_no data = dict(id=seq_no, method=method, params=params) q = Queue() self._response_queues[seq_no] = q try: self._send_command(json.dumps(data)) return q.get(timeout=self.TIMEOUT) except Empty as e: if self.is_connected(): self.logger.exception('method={} params={}\n{}'.format( method, params, str(e))) raise return dict(error='response timeout by disconnection') except Exception as e: self.logger.exception('method={} params={}\n{}'.format( method, params, str(e))) raise finally: del self._response_queues[seq_no]
class _Channel(object): def __init__(self, from_fsm, to_fsm, tracer, queue=None): if queue is None: self.queue = Queue() else: self.queue = queue self.from_fsm = from_fsm self.to_fsm = to_fsm self.tracer = tracer def put(self, item): self.tracer.send_trace_message( messages.ChannelTrace( self.tracer.trace_order_seq(), self.from_fsm.fsm_id if self.from_fsm else None, self.to_fsm.fsm_id if self.to_fsm else None, item.__class__.__name__)) self.queue.put(item) def get(self, block=True, timeout=None): return self.queue.get(block, timeout) receive = get
class Actor(gevent.Greenlet): def __init__(self): self.inbox = Queue() gevent.Greenlet.__init__(self) def on_message(self, message, answer_box=None): """ Define in your subclass. """ raise NotImplemented() def send_to(self, dest, message): mail = Mail(self.inbox, message) if isinstance(dest, Actor): dest = dest.inbox dest.put(mail) def _run(self): self.running = True while self.running: message = self.inbox.get() if isinstance(message, Mail): self.on_message(message.body, answer_box=message.answer_box) else: self.on_message(message)
def __bfs(self): q = Queue() # 消费者 def dispatch(step=0, args=tuple()): job = self.jobs[step] job_name = '%s%s' % (job.__name__, args) logger.info('Dispatching jobs of %s.', job_name) rv = job(*args) if rv: step += 1 for next_args in rv: q.put((step, next_args)) logger.info('Jobs of %s have been dispatched.', job_name) # 生产者 q.put((0, tuple())) while True: while not q.empty(): self.pool.spawn(dispatch, *q.get()) self.pool.join() if q.empty(): break
class Importer(gevent.Greenlet): def __init__(self): self.inbox = Queue() self._es = Elasticsearch({ "host": config['es_host'], "port": config['es_port'] }) Greenlet.__init__(self) def _process(self, message): if message == "new_content": content_importer = ContentLocalImporter() content_importer.download_files() else: helpers.bulk(self._es, message) def _run(self): self.running = True while self.running: message = self.inbox.get() self._process(message)
class GCMService(BaseService): def __init__(self, api_key): self.feedback_queue = Queue() self.workers = [ GCMServiceWorker(i, api_key, self.feedback_queue) for i in range(WORKER_COUNT) ] self.next_worker = 0 def get_feedback(self, block=True, timeout=None): return self.feedback_queue.get(block=block, timeout=timeout) def queue_notification(self, notification): self.workers[self.next_worker].queue_notification(notification) self.next_worker = (self.next_worker + 1) % WORKER_COUNT def start(self): for w in self.workers: w.start() def stop(self, timeout=10.0): for w in self.workers: w.stop()
def test_concurrent_set_total_deposit( token_network_proxy: TokenNetwork) -> None: CHANNEL_COUNT = 3 DEPOSIT_COUNT = 5 channels = Queue() def open_channel() -> None: partner = factories.make_address() settle_timeout = 500 given_block_identifier = BLOCK_ID_LATEST channel_identifier, _, block_hash = token_network_proxy.new_netting_channel( partner, settle_timeout, given_block_identifier) channels.put((channel_identifier, block_hash, partner)) channel_grenlets = { gevent.spawn(open_channel) for _ in range(CHANNEL_COUNT) } deposit_greenlets = set() for _ in range(CHANNEL_COUNT): channel_identifier, block_hash, partner = channels.get() for i in range(DEPOSIT_COUNT): given_block_identifier = block_hash total_deposit = i + 1 g = gevent.spawn( token_network_proxy.approve_and_set_total_deposit, given_block_identifier, channel_identifier, total_deposit, partner, ) deposit_greenlets.add(g) all_greenlets: Set[Greenlet] = channel_grenlets.union(deposit_greenlets) gevent.joinall(set(all_greenlets), raise_error=True)
class GeventActor(gevent.Greenlet): def __init__(self, interval, name=''): super(GeventActor, self).__init__() self.inbox = G_Queue() # AsyncResult self._interval = interval self._timeout = None self.name = name def _work(self): """ Define in your subclass. """ raise NotImplemented() def _run(self): while True: try: msg = self.inbox.get(block=True, timeout=self._timeout) if msg == 'stop': self._timeout = None elif msg == 'start': self._timeout = self._interval elif msg == 'exit': break except G_Empty: self._work() gevent.sleep(0) def start(self): self.inbox.put('start') return super(GeventActor, self).start() def stop(self): self.inbox.put('stop') def shutdown(self): self.inbox.put('exit')