def getTweetsMulti(self, userids):
foundTweets = {}
badIds = []
nids = len(userids)
pool = eventlet.GreenPool()
ngotten = 0
notFound = copy(userids)
returned = False
for tweettag in pool.imap(self.getTweetsMultiHelper, userids):
if returned:
eventlet.kill()
ngotten += 1
tweets, tag, id = tweettag
if tag:
foundTweets[id] = tweets
else:
badIds.append(id)
notFound.remove(id)
if ngotten > nids * .8:
returned = True
return foundTweets, badIds, notFound
return foundTweets, badIds, notFound
def _spin_container_and_wait(self, sensors):
exit_code = 0
try:
self._sensor_container = ProcessSensorContainer(
sensors=sensors, single_sensor_mode=self._single_sensor_mode)
self._container_thread = eventlet.spawn(self._sensor_container.run)
LOG.debug('Starting sensor CUD watcher...')
self._sensors_watcher.start()
exit_code = self._container_thread.wait()
LOG.error('Process container quit with exit_code %d.', exit_code)
LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
except (KeyboardInterrupt, SystemExit):
self._sensor_container.shutdown()
self._sensors_watcher.stop()
LOG.info('(PID:%s) SensorContainer stopped. Reason - %s',
os.getpid(),
sys.exc_info()[0].__name__)
eventlet.kill(self._container_thread)
self._container_thread = None
return exit_code
return exit_code
def test_killing_dormant(self):
DELAY = 0.1
state = []
def test():
try:
state.append('start')
eventlet.sleep(DELAY)
except:
state.append('except')
# catching GreenletExit
pass
# when switching to hub, hub makes itself the parent of this greenlet,
# thus after the function's done, the control will go to the parent
eventlet.sleep(0)
state.append('finished')
g = eventlet.spawn(test)
eventlet.sleep(DELAY / 2)
self.assertEqual(state, ['start'])
eventlet.kill(g)
# will not get there, unless switching is explicitly scheduled by kill
self.assertEqual(state, ['start', 'except'])
eventlet.sleep(DELAY)
self.assertEqual(state, ['start', 'except', 'finished'])
def __del__(self):
# This function gets called when the interpreter deletes the object
# by the automatic garbage cleanup
for gt in self.greenThreads:
eventlet.kill(gt)
eventlet.kill(self)
def __del__(self):
# This function gets called when the interpreter deletes the object
# by the automatic garbage cleanup
for gt in self.greenThreads:
eventlet.kill(gt)
eventlet.kill(self)
def test_blocks_on_pool(self):
waiter = Queue(0)
def greedy():
self.pool.get()
self.pool.get()
self.pool.get()
self.pool.get()
# No one should be waiting yet.
self.assertEquals(self.pool.waiting(), 0)
# The call to the next get will unschedule this routine.
self.pool.get()
# So this put should never be called.
waiter.put('Failed!')
killable = eventlet.spawn(greedy)
# no one should be waiting yet.
self.assertEquals(self.pool.waiting(), 0)
## Wait for greedy
eventlet.sleep(0)
## Greedy should be blocking on the last get
self.assertEquals(self.pool.waiting(), 1)
## Send will never be called, so balance should be 0.
self.assertFalse(not waiter.full())
eventlet.kill(killable)
def _spin_container_and_wait(self, sensors):
exit_code = 0
try:
self._sensor_container = ProcessSensorContainer(
sensors=sensors,
single_sensor_mode=self._single_sensor_mode)
self._container_thread = eventlet.spawn(self._sensor_container.run)
LOG.debug('Starting sensor CUD watcher...')
self._sensors_watcher.start()
exit_code = self._container_thread.wait()
LOG.error('Process container quit with exit_code %d.', exit_code)
LOG.error('(PID:%s) SensorContainer stopped.', os.getpid())
except (KeyboardInterrupt, SystemExit):
self._sensor_container.shutdown()
self._sensors_watcher.stop()
LOG.info('(PID:%s) SensorContainer stopped. Reason - %s', os.getpid(),
sys.exc_info()[0].__name__)
eventlet.kill(self._container_thread)
self._container_thread = None
return exit_code
return exit_code
def test_timeout_cancel(self):
server = eventlet.listen(('0.0.0.0', 0))
bound_port = server.getsockname()[1]
done = [False]
def client_closer(sock):
while True:
(conn, addr) = sock.accept()
conn.close()
def go():
desc = eventlet.connect(('127.0.0.1', bound_port))
try:
hubs.trampoline(desc, read=True, timeout=0.1)
except eventlet.TimeoutError:
assert False, "Timed out"
server.close()
desc.close()
done[0] = True
greenthread.spawn_after_local(0, go)
server_coro = eventlet.spawn(client_closer, server)
while not done[0]:
eventlet.sleep(0)
eventlet.kill(server_coro)
check_hub()
def close(self):
self._is_running = False
if self._uvc_thread:
try:
eventlet.kill(self._uvc_thread)
except eventlet.greenlet.GreenletExit:
pass
self._uvc_thread = None
def run(self):
self.socket = GreenSocket(family_or_realsock=self.socket.sock)
self.socket.setblocking(1)
self.acceptor = eventlet.spawn(eventlet.serve, self.socket, server.handle, int(config.get("MAX_CONNECTIONS")))
while True:
eventlet.sleep(0.1)
with eventlet.Timeout(30, False):
eventlet.kill(self.acceptor, eventlet.StopServe)
def _stop_checker(self, green_thread):
""" Propagates exceptions to the run green thread.
"""
try:
green_thread.wait()
except greenlet.GreenletExit: # pylint: disable=no-member
pass
except Exception: # pylint: disable=broad-except
eventlet.kill(self.__run_gt, *sys.exc_info())
def close(self):
while self._threads:
eventlet.kill( self._threads.pop() )
self._socket = None
for service in self.services:
try:
service.withdraw()
except:
pass
def _cleanup(self):
if self._active_loop:
loop = self._active_loop
self._active_loop = None
eventlet.kill(loop)
if self._sock:
sock = self._sock
self._sock = None
sock.close()
def _stop_checker(self, green_thread):
""" Propagates exceptions to the run green thread.
"""
try:
green_thread.wait()
except greenlet.GreenletExit: # pylint: disable=no-member
pass
except Exception: # pylint: disable=broad-except
eventlet.kill(self.__run_gt, *sys.exc_info())
def finish(self):
if not self._running:
return
self._dout.close()
if(self._recvWorker):
self._recvWorker.finish()
_sender_worker_map.pop(self._sid)
self._running = False
kill(self.id)
log.debug('finish send worker with sid %d', self._sid)
def run(self):
self.socket = GreenSocket(family_or_realsock=self.socket.sock)
self.socket.setblocking(1)
self.acceptor = eventlet.spawn(eventlet.serve, self.socket,
server.handle,
int(config.get("MAX_CONNECTIONS")))
while True:
eventlet.sleep(0.1)
with eventlet.Timeout(30, False):
eventlet.kill(self.acceptor, eventlet.StopServe)
def handle_client_disconnect():
"""Handle client disconnected.
Remove subscription threads.
"""
for lgt in self.listen_gthreads[request.sid]:
eventlet.kill(lgt)
for sub in self.subscriptions[request.sid]:
self.pubsub.unsubscribe(sub)
log.info(f"Client {current_user} disconnected")
def stop(self):
""" 结束事件循环 """
if self.started:
self.started = False
self.looping_call.stop()
eventlet.kill(self.loop_thread,
exception.StopEventLoop)
self.looping_call = None
self.loop_thread = None
def _ripper(self):
# Not sure if this will ever be called, but for correctness
# sake, thought I'd add it
while True:
if self._should_stop:
if len(self._eventlets):
for greenthread in self._eventlets:
eventlet.kill(greenthread)
self._eventlets = []
return
time.sleep(.5)
def __stop_checker(self, green_thread):
""" Propagates exceptions to the dispatcher green thread.
"""
ifindex = None
try:
ifindex = green_thread.wait()
except greenlet.GreenletExit: # pylint: disable=no-member
pass
except Exception: # pylint: disable=broad-except
eventlet.kill(self.dispatcher_gt, *sys.exc_info())
self.__running.remove(ifindex)
self.__intfevent.put(ifindex)
def __stop_checker(self, green_thread):
""" Propagates exceptions to the dispatcher green thread.
"""
ifindex = None
try:
ifindex = green_thread.wait()
except greenlet.GreenletExit: # pylint: disable=no-member
pass
except Exception: # pylint: disable=broad-except
eventlet.kill(self.dispatcher_gt, *sys.exc_info())
self.__running.remove(ifindex)
self.__intfevent.put(ifindex)
def test_urllib():
gthread, server, port = start_http_server()
try:
assert server.request_count == 0
try:
urlopen('http://127.0.0.1:{0}'.format(port))
assert False, 'should not get there'
except HTTPError as ex:
assert ex.code == 501, repr(ex)
assert server.request_count == 1
finally:
server.server_close()
eventlet.kill(gthread)
def test_urllib():
gthread, server, port = start_http_server()
try:
assert server.request_count == 0
try:
urlopen('http://127.0.0.1:{0}'.format(port))
assert False, 'should not get there'
except HTTPError as ex:
assert ex.code == 501, repr(ex)
assert server.request_count == 1
finally:
server.server_close()
eventlet.kill(gthread)
def kill(self):
"""
杀死所有进程，包括协程。
:return:
"""
for i in self.proc_list:
try:
i.kill()
except OSError:
pass
del self.proc_list
for i in self.gt_list:
eventlet.kill(i)
def main_process_stop():
"""
Verifies if main process running and if it is - terminates it using process.join() method with timeout = 5 seconds.
After it returns status of this process (should be terminated).
:return: main_process_status()
"""
global MAIN_PROCESS
if MAIN_PROCESS:
MAIN_PROCESS.cancel()
eventlet.kill(MAIN_PROCESS)
MAIN_PROCESS = None
time.sleep(0.5)
return main_process_status()
def stop(self):
try:
if self._updates_thread:
self._updates_thread = eventlet.kill(self._updates_thread)
finally:
if self.connection:
self.connection.release()
def run(self):
self.socket = GreenSocket(family_or_realsock=self.socket.sock)
self.socket.setblocking(1)
self.acceptor = eventlet.spawn(eventlet.serve, self.socket,
self.handle, self.worker_connections)
while self.alive:
self.notify()
if self.ppid != os.getppid():
self.log.info("Parent changed, shutting down: %s" % self)
break
eventlet.sleep(0.1)
self.notify()
with eventlet.Timeout(self.timeout, False):
eventlet.kill(self.acceptor, eventlet.StopServe)
def stop(self):
try:
if self._updates_thread:
self._updates_thread = eventlet.kill(self._updates_thread)
finally:
if self.connection:
self.connection.release()
def run(self) -> NoReturn:
"""Spawns a greenthread subscribing to PostgreSQL database
notification channel and waits for NOTIFYs. On receipt the thread is
blocked and the Listener.handler() is fired. Once the handler has
finished processing the thread is finally unblocked and waits for
the next NOTIFY.
"""
# multi-producer, multi-consumer queue that works across greenlets
# size 1 (i.e. not infinite) so that it blocks until entry processed
queue = eventlet.Queue(0)
# spawn async greenthread in parallel
g = eventlet.spawn(self.subscribe, queue)
while True:
try:
logger.debug(f"Waiting for a notification...")
notify = queue.get()
# ^ blocks until item available in queue. i.e. waiting for
# spawned function to yield
# ----------------%--------------------%----------------------#
logger.info(f"Got NOTIFY: "
f"{notify.pid} {notify.channel} {notify.payload}")
# do something with the database once received the NOTIFY (n)
self.handler.on_notify()
# tell queue that this consumer has finished the task for
# which it asked q.get()
queue.task_done()
# block until all items in the queue have been gotten and
# processed:
queue.join()
except KeyboardInterrupt:
eventlet.kill(g)
logger.error("Listener has been killed via Keyboard "
"Interrupt. Greenthread garbage collected.")
break
def getNameMulti(self, pageids):
foundNames = {}
badIds = []
nids = len(pageids)
pool = eventlet.GreenPool()
ngotten = 0
returned = False
for nametag in pool.imap(self.getNameMultiHelper, pageids):
if returned:
eventlet.kill()
ngotten += 1
name, tag = nametag
if tag:
id, username, popularity, importance, birthtime = name
foundNames[id] = (username, popularity, importance, birthtime)
else:
badIds.append(name)
if ngotten > nids * .8:
returned = True
return foundNames, badIds
return foundNames, badIds
def getFollowersMulti(self, pageids, maxpage=9999):
self.maxpage = maxpage
foundFollowers = []
badIds = []
pool = eventlet.GreenPool()
hasReturned = False
ngotten = 0
nids = len(pageids)
for idtag in pool.imap(self.getFollowersMultiHelper, pageids):
if hasReturned:
eventlet.kill()
ids, tag = idtag
if tag:
foundFollowers.append(ids)
ngotten += 1
else:
badIds.append(ids)
if ngotten > nids * .5:
hasReturned = True
return foundFollowers, badIds
return foundFollowers, badIds
def run(self):
self.socket = GreenSocket(family_or_realsock=self.socket.sock)
self.socket.setblocking(1)
pool = eventlet.GreenPool(self.worker_connections)
acceptor = eventlet.spawn(self.acceptor, pool)
try:
while self.alive:
self.notify()
if self.ppid != os.getppid():
self.log.info("Parent changed, shutting down: %s" % self)
server.stop()
break
eventlet.sleep(0.1)
except KeyboardInterrupt:
pass
with eventlet.Timeout(self.timeout, False):
eventlet.kill(acceptor, eventlet.StopServe)
def run(self):
queue = eventlet.Queue() # multi-producer, multi-consumer queue that works across greenlets
g = eventlet.spawn(self.subscribe, queue) # spawn async greenthread in parallel
while True:
try:
logger.debug(f"Waiting for a notification...")
notify = queue.get() # blocks until item available in queue
# -------------%------------------%--------------------%-------------------#
logger.success(f"Got NOTIFY: {notify.pid} {notify.channel} {notify.payload}")
# do something once received the NOTIFY (n)
self.handler.on_notify()
queue.task_done() # tell queue that this consumer has finished the task for which it asked q.get()
except (Exception, KeyboardInterrupt):
eventlet.kill(g)
logger.error("Listener has been killed via Keyboard Interrupt. Greenthread garbage collected.")
break
def stop(self):
if self.started:
self.started = False
self.price_updater.stop()
eventlet.kill(self.recv_loop, exception.StopEventLoop)
self.recv_loop = None
eventlet.kill(self.event_thread, exception.StopEventLoop)
self.event_thread = None
eventlet.kill(self.send_thread, exception.StopEventLoop)
self.send_thread = None
def stop(self):
LOG.debug('Shutting down sensor watcher.')
try:
if self._updates_thread:
self._updates_thread = eventlet.kill(self._updates_thread)
if self.connection:
channel = self.connection.channel()
bound_sensor_watch_q = self._sensor_watcher_q(channel)
try:
bound_sensor_watch_q.delete()
except:
LOG.error('Unable to delete sensor watcher queue: %s', self._sensor_watcher_q)
finally:
if self.connection:
self.connection.release()
def run(self):
acceptors = []
for sock in self.sockets:
s = GreenSocket(family_or_realsock=sock)
s.setblocking(1)
hfun = partial(self.handle, s)
acceptor = eventlet.spawn(eventlet.serve, s, hfun,
self.worker_connections)
while self.alive:
self.notify()
if self.ppid != os.getppid():
self.log.info("Parent changed, shutting down: %s", self)
break
eventlet.sleep(1.0)
self.notify()
with eventlet.Timeout(self.cfg.graceful_timeout, False):
[eventlet.kill(a, eventlet.StopServe) for a in acceptors]
def force_finish(self):
kill(self._recvWorker.id)
def stop(self):
try:
self._thread = eventlet.kill(self._thread)
finally:
self.connection.release()
def __term_handler(self, signum, _):
""" Signal handler that terminates the process.
"""
msg = 'Caught signal %d' % signum
self._logger.debug(msg)
eventlet.kill(self.__run_gt, SystemExit(msg))
def stop(self):
self._shutdown = True
return eventlet.kill(self._flush_thread)
def tearDown(self):
self.server.server_close()
kill(self.gthread)
def kill(self):
"""Violently kill the Actor at this Address. Any other Actor which has
called wait on this Address will get a Killed exception.
"""
eventlet.kill(self._actor, Killed)
def __term_handler(self, signum, _):
""" Signal handler that terminates the process.
"""
msg = 'Caught signal %d' % signum
self._logger.debug(msg)
eventlet.kill(self.__run_gt, SystemExit(msg))
def stop(self):
self._stop= True
kill(self.id)
def kill(self):
for thread in self.threads:
eventlet.kill(thread)
self.is_runned = False
