def test_with_timeout(self):
self.assertRaises(eventlet.Timeout, eventlet.with_timeout, DELAY, eventlet.sleep, DELAY * 10)
X = object()
r = eventlet.with_timeout(DELAY, eventlet.sleep, DELAY * 10, timeout_value=X)
assert r is X, (r, X)
r = eventlet.with_timeout(DELAY * 10, eventlet.sleep, DELAY, timeout_value=X)
assert r is None, r
def test_defer_event(ctx):
from datetime import datetime, timedelta
from eventlet import sleep, spawn, with_timeout
from eventlet.event import Event
from melkman.messaging import EventBus
from melkman.scheduler import defer_event
from melkman.scheduler.worker import ScheduledMessageService
CHAN = 'test_chan'
sms = ScheduledMessageService(ctx)
sched = spawn(sms.run)
got_message = Event()
def got_message_cb(*args, **kw):
got_message.send(True)
eb = EventBus(ctx)
eb.add_listener(CHAN, got_message_cb)
now = datetime.utcnow()
wait = timedelta(seconds=2)
defer_event(now + wait, CHAN, {'foo': 'bar'}, ctx)
sleep(3)
try:
with_timeout(10, got_message.wait)
assert got_message.ready()
finally:
eb.kill()
sched.kill()
sched.wait()
def check_timed_out(self, event, myproc, proc_finished_flag, queue):
X = object()
assert with_timeout(DELAY, event.wait, timeout_value=X) is X
assert with_timeout(DELAY, queue.wait, timeout_value=X) is X
assert with_timeout(DELAY, proc.waitall, [myproc],
timeout_value=X) is X
assert proc_finished_flag == [], proc_finished_flag
def serve_from_child(sock, config, controller_pid):
threads = config.get('threadpool_workers', 0)
wsgi_application = spawning.util.named(config['app_factory'])(config)
if config.get('coverage'):
wsgi_application = FigleafCoverage(wsgi_application)
if config.get('sysinfo'):
wsgi_application = SystemInfo(wsgi_application)
if threads >= 1:
# proxy calls of the application through tpool
wsgi_application = tpool_wsgi(wsgi_application)
elif threads != 1:
from eventlet.green import socket
host, port = sock.getsockname()
access_log_file = config.get('access_log_file')
if access_log_file is not None:
access_log_file = open(access_log_file, 'a')
max_age = 0
if config.get('max_age'):
max_age = int(config.get('max_age'))
server_event = eventlet.event.Event()
# the status object wants to have a reference to the server object
if config.get('status_port'):
def send_server_to_status(server_event):
server = server_event.wait()
get_statusobj().server = server
eventlet.spawn(send_server_to_status, server_event)
http_version = config.get('no_keepalive') and 'HTTP/1.0' or 'HTTP/1.1'
try:
wsgi_args = (sock, wsgi_application)
wsgi_kwargs = {'log' : access_log_file, 'server_event' : server_event, 'max_http_version' : http_version}
if config.get('no_keepalive'):
wsgi_kwargs.update({'keepalive' : False})
if max_age:
wsgi_kwargs.update({'timeout_value' : True})
eventlet.with_timeout(max_age, eventlet.wsgi.server, *wsgi_args,
**wsgi_kwargs)
warn_controller_of_imminent_death(controller_pid)
else:
eventlet.wsgi.server(*wsgi_args, **wsgi_kwargs)
except KeyboardInterrupt:
# controller probably doesn't know that we got killed by a SIGINT
warn_controller_of_imminent_death(controller_pid)
except ExitChild:
pass # parent killed us, it already knows we're dying
## Once we get here, we should not accept any new sockets, so we should close the server socket.
sock.close()
server = server_event.wait()
print "(%s) *** Child exiting" % (os.getpid(),)
os.kill(os.getpid(), signal.SIGKILL)
def test_nested_with_timeout(self):
def func():
return eventlet.with_timeout(0.2, eventlet.sleep, 2, timeout_value=1)
try:
eventlet.with_timeout(0.1, func)
self.fail(u'Expected TimeoutError')
except eventlet.TimeoutError:
pass
def test_nested_with_timeout(self):
def func():
return eventlet.with_timeout(0.2, eventlet.sleep, 2, timeout_value=1)
try:
eventlet.with_timeout(0.1, func)
self.fail(u'Expected Timeout')
except eventlet.Timeout:
pass
def stop(self):
self._stopped = True
if self._greenlet and not self._greenlet.dead:
self._greenlet.kill(ElectorStopped(), None, None)
try:
# It should die very quickly.
eventlet.with_timeout(10, self._greenlet.wait)
except eventlet.Timeout:
# Looks like we've leaked the greenlet somehow.
_log.error("Timeout while waiting for the greenlet to die.")
raise RuntimeError("Failed to kill Elector greenlet.")
except ElectorStopped:
pass # Expected
def test_deferred_send_receive(ctx):
from datetime import datetime, timedelta
from carrot.messaging import Consumer
from eventlet import sleep, spawn, with_timeout
from eventlet.event import Event
from eventlet.support.greenlets import GreenletExit
import logging
from melk.util.nonce import nonce_str
import sys
from melkman.context import Context
from melkman.scheduler import defer_amqp_message, cancel_deferred
from melkman.scheduler.worker import ScheduledMessageService
got_message = Event()
def got_message_cb(*args, **kw):
got_message.send(True)
def do_consume():
consumer = Consumer(ctx.broker, exchange='testx', queue='testq',
routing_key='testq', exclusive=True, durable=False)
consumer.register_callback(got_message_cb)
try:
consumer.wait(limit=1)
except StopIteration:
pass
except GreenletExit:
pass
finally:
consumer.close()
cons = spawn(do_consume)
sms = ScheduledMessageService(ctx)
sched = spawn(sms.run)
m1 = {'hello': 'world'}
now = datetime.utcnow()
wait = timedelta(seconds=2)
defer_amqp_message(now + wait, m1, 'testq', 'testx', ctx)
try:
#sleep(1)
with_timeout(10, got_message.wait)
assert got_message.ready()
finally:
sched.kill()
sched.wait()
cons.kill()
cons.wait()
def test_with_timeout(self):
self.assertRaises(eventlet.Timeout, eventlet.with_timeout, DELAY,
eventlet.sleep, DELAY * 10)
X = object()
r = eventlet.with_timeout(DELAY,
eventlet.sleep,
DELAY * 10,
timeout_value=X)
assert r is X, (r, X)
r = eventlet.with_timeout(DELAY * 10,
eventlet.sleep,
DELAY,
timeout_value=X)
assert r is None, r
def _wait_and_stop(self, client, elector):
# Wait for the client to tell us that all the results have been
# processed.
try:
eventlet.with_timeout(5, client.no_more_results.wait)
except eventlet.Timeout:
elector._greenlet.kill(AssertionError("Didn't reach end of results"))
elector._greenlet.wait()
# This should shut down the Elector.
eventlet.with_timeout(5, elector.stop)
# The greenlet should be dead already, but just in case, let our
# client proceed to raise its exception.
client.stop.send()
# Double-check there were no failures.
self.assertEqual(client.failure, None, msg=client.failure)
def read(self, path, **kwargs):
try:
result = self.results.pop(0)
except IndexError:
if not self.no_more_results.ready():
self.no_more_results.send()
eventlet.with_timeout(5, self.stop.wait)
raise NoMoreResults()
if result.op != READ:
self.failure = "Unexpected result type for read(): %s" % result.op
raise UnexpectedResultType()
if result.exception is not None:
log.debug("Raise read exception %s", type(result.exception).__name__)
raise result.exception
log.debug("Return read result %s", result)
return result
def test_blocking(self):
l = eventlet.listen(('localhost', 0))
x = eventlet.with_timeout(
0.01,
eventlet.serve, l, lambda c, a: None,
timeout_value="timeout")
self.assertEqual(x, "timeout")
def next(self):
"""
Implements the iterator protocol for `AndroidMarketCrawler`
(see usage example above)
"""
# when there are results, then return them even though you've
# got other things to do, too
if not self.results.empty():
return self.results.get()
# as long as there are tasks scheduled in the queue, or as
# long as there are active scripts running ...
while not self.queue.empty() or self.pool.running() != 0:
# gets a new URL from the queue, to be fetched. if the
# queue is empty, then waits until it isn't (eventlet's
# run-loop can continue processing during this wait)
url = eventlet.with_timeout(0.1, self.queue.get, timeout_value='')
# if we have a new URL, then we spawn another green thread for fetching the content
if url:
if url in self.seen: continue
uid = self.get_id(url)
if uid in self.seen_app_ids: continue
self.seen.add(url)
self.pool.spawn_n(self.fetch_content, url)
# in case we have results waiting to be served, then
# return
if not self.results.empty():
return self.results.get()
raise StopIteration
def start(self):
while not self.url_queue.empty() or self.pool.running() != 0:
print self.url_queue.empty()
url = eventlet.with_timeout(0.1, self.url_queue.get, timeout_value='')
# limit requests to eventlet.net so we don't crash all over the internet
if url:
self.pool.spawn_n(self.__fetch_url, url)
def read(self, path, **kwargs):
try:
result = self.results.pop(0)
except IndexError:
if not self.no_more_results.ready():
self.no_more_results.send()
eventlet.with_timeout(5, self.stop.wait)
raise NoMoreResults()
if result.op != READ:
self.failure = "Unexpected result type for read(): %s" % result.op
raise UnexpectedResultType()
if result.exception is not None:
log.debug("Raise read exception %s", type(result.exception).__name__)
raise result.exception
log.debug("Return read result %s", result)
return result
def test_pause_producing(self):
self.conn.pauseProducing()
self.conn.write('hi\r\n')
result = with_timeout(DELAY * 10,
self.conn.read,
timeout_value='timed out')
self.assertEqual('timed out', result)
def test_blocking(self):
l = eventlet.listen(('localhost', 0))
x = eventlet.with_timeout(
0.01,
eventlet.serve, l, lambda c, a: None,
timeout_value="timeout")
self.assertEqual(x, "timeout")
def test_defer_message_dispatch(ctx):
from datetime import datetime, timedelta
from eventlet import sleep, spawn, with_timeout
from eventlet.event import Event
from melkman.messaging import MessageDispatch, always_ack
from melkman.scheduler import defer_message
from melkman.scheduler.worker import ScheduledMessageService
sms = ScheduledMessageService(ctx)
sched = spawn(sms.run)
w = MessageDispatch(ctx)
message_type = 'test_dispatch_send_recv'
work_result = Event()
@always_ack
def handler(job, message):
work_result.send(sum(job['values']))
worker = w.start_worker(message_type, handler)
try:
now = datetime.utcnow()
wait = timedelta(seconds=2)
# w.send({'values': [1, 2]}, message_type)
defer_message(now + wait, {'values': [1 ,2]}, message_type, ctx)
sleep(3)
assert with_timeout(2, work_result.wait) == 3
finally:
worker.kill()
worker.wait()
sched.kill()
sched.wait()
def run_loop(self):
"""Runs io-worker until it dies.
You SHOULD spawn this function (so it runs in separate thread).
"""
args = ["io-worker/io-worker", "-skip-robots"]
self.worker = subprocess.Popen(args, bufsize=1,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
while not self.is_closed():
encoded = with_timeout(0.050, _io_op, self.worker.stdout.readline,
timeout_value="")
encoded = encoded.strip()
if not encoded:
continue
url, result = _parse(encoded)
if url is None:
continue
if url in self.results:
out = self.results.pop(url)
out.send(result)
else:
log.error("Got result for non-requested url: %s.", url)
self.worker.stdin.close()
def _wait_and_stop(self, client, elector):
# Wait for the client to tell us that all the results have been
# processed.
try:
eventlet.with_timeout(5, client.no_more_results.wait)
except eventlet.Timeout:
elector._greenlet.kill(
AssertionError("Didn't reach end of results"))
elector._greenlet.wait()
raise
# This should shut down the Elector.
eventlet.with_timeout(5, elector.stop)
# The greenlet should be dead already, but just in case, let our
# client proceed to raise its exception.
client.stop.send()
# Double-check there were no failures.
self.assertIsNone(client.failure, msg=client.failure)
def write(self, path, value, **kwargs):
log.debug("Write of %s to %s", value, path)
try:
result = self.results.pop(0)
except IndexError:
if not self.no_more_results.ready():
self.no_more_results.send()
eventlet.with_timeout(5, self.stop.wait)
raise NoMoreResults()
if result.op != WRITE:
self.failure = "Unexpected result type for write(): %s" % result.op
raise UnexpectedResultType()
if result.exception is not None:
log.debug("Raise write exception %s", result.exception)
raise result.exception
log.debug("Return write result")
return result
def test_pauseresume_producing(self):
self.conn.pauseProducing()
spawn_after(DELAY * 5, self.conn.resumeProducing)
self.conn.write('hi\r\n')
result = with_timeout(DELAY * 10,
self.conn.read,
timeout_value='timed out')
self.assertEqual('you said hi. BYE', result)
def test_communicate_with_poll():
# https://github.com/eventlet/eventlet/pull/24
# `eventlet.green.subprocess.Popen.communicate()` was broken
# in Python 2.7 because the usage of the `select` module was moved from
# `_communicate` into two other methods `_communicate_with_select`
# and `_communicate_with_poll`. Link to 2.7's implementation:
# http://hg.python.org/cpython/file/2145593d108d/Lib/subprocess.py#l1255
if getattr(original_subprocess.Popen, '_communicate_with_poll', None) is None:
raise SkipTest('original subprocess.Popen does not have _communicate_with_poll')
p = subprocess.Popen(
[sys.executable, '-c', 'import time; time.sleep(0.5)'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
t1 = time.time()
eventlet.with_timeout(0.1, p.communicate, timeout_value=True)
tdiff = time.time() - t1
assert 0.1 <= tdiff <= 0.2, 'did not stop within allowed time'
def write(self, path, value, **kwargs):
log.debug("Write of %s to %s", value, path)
try:
result = self.results.pop(0)
except IndexError:
if not self.no_more_results.ready():
self.no_more_results.send()
eventlet.with_timeout(5, self.stop.wait)
raise NoMoreResults()
if result.op != WRITE:
self.failure = "Unexpected result type for write(): %s" % result.op
raise UnexpectedResultType()
if result.exception is not None:
log.debug("Raise write exception %s", result.exception)
raise result.exception
log.debug("Return write result")
return result
def test_communicate_with_poll():
# This test was being skipped since git 25812fca8, I don't there's
# a need to do this. The original comment:
#
# https://github.com/eventlet/eventlet/pull/24
# `eventlet.green.subprocess.Popen.communicate()` was broken
# in Python 2.7 because the usage of the `select` module was moved from
# `_communicate` into two other methods `_communicate_with_select`
# and `_communicate_with_poll`. Link to 2.7's implementation:
# http://hg.python.org/cpython/file/2145593d108d/Lib/subprocess.py#l1255
p = subprocess.Popen(
[sys.executable, '-c', 'import time; time.sleep(0.5)'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
t1 = time.time()
eventlet.with_timeout(0.1, p.communicate, timeout_value=True)
tdiff = time.time() - t1
assert 0.1 <= tdiff <= 0.2, 'did not stop within allowed time'
def run_dispatcher(self):
try:
# cleanup any mess left over last time...
with self.context:
self.cleanup()
while(True):
log.info("checking for ready messages...")
last_time = self.send_ready_messages()
sleep_time = self._calc_sleep(last_time)
log.info("sleeping for %s" % sleep_time)
sleep_secs = sleep_time.days*84600 + sleep_time.seconds
try:
with_timeout(sleep_secs, self.service_queue.wait)
except TimeoutError:
pass
if self.service_queue.ready():
self.service_queue.reset()
except GreenletExit:
log.debug("ScheduledMessageService dispatcher exiting...")
def test_communicate_with_poll():
# https://github.com/eventlet/eventlet/pull/24
# `eventlet.green.subprocess.Popen.communicate()` was broken
# in Python 2.7 because the usage of the `select` module was moved from
# `_communicate` into two other methods `_communicate_with_select`
# and `_communicate_with_poll`. Link to 2.7's implementation:
# http://hg.python.org/cpython/file/2145593d108d/Lib/subprocess.py#l1255
if getattr(original_subprocess.Popen, '_communicate_with_poll',
None) is None:
raise SkipTest(
'original subprocess.Popen does not have _communicate_with_poll')
p = subprocess.Popen(
[sys.executable, '-c', 'import time; time.sleep(0.5)'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
t1 = time.time()
eventlet.with_timeout(0.1, p.communicate, timeout_value=True)
tdiff = time.time() - t1
assert 0.1 <= tdiff <= 0.2, 'did not stop within allowed time'
def test_send(self):
event1 = Event()
event2 = Event()
spawn(event1.send, 'hello event1')
eventlet.Timeout(0, ValueError('interrupted'))
try:
result = event1.wait()
except ValueError:
X = object()
result = with_timeout(DELAY, event2.wait, timeout_value=X)
assert result is X, 'Nobody sent anything to event2 yet it received %r' % (result, )
def test_send(self):
event1 = Event()
event2 = Event()
spawn(event1.send, 'hello event1')
eventlet.Timeout(0, ValueError('interrupted'))
try:
result = event1.wait()
except ValueError:
X = object()
result = with_timeout(DELAY, event2.wait, timeout_value=X)
assert result is X, 'Nobody sent anything to event2 yet it received %r' % (
result, )
def producer(start_url):
"""Recursively crawl starting from *start_url*. Returns a set of
urls that were found."""
pool = eventlet.GreenPool()
seen = set()
q = eventlet.Queue()
q.put(start_url)
# keep looping if there are new urls, or workers that may produce more urls
while not q.empty() or pool.running() != 0:
url = eventlet.with_timeout(0.1, q.get, timeout_value='')
# limit requests to eventlet.net so we don't crash all over the internet
if url not in seen and 'eventlet.net' in url:
seen.add(url)
pool.spawn_n(fetch, url, q)
return seen
def test_concurrency(self):
evt = event.Event()
def waiter(sock, addr):
sock.sendall(s2b('hi'))
evt.wait()
l = eventlet.listen(('localhost', 0))
gt = eventlet.spawn(eventlet.serve, l, waiter, 5)
def test_client():
c = eventlet.connect(('localhost', l.getsockname()[1]))
# verify the client is connected by getting data
self.assertEquals(s2b('hi'), c.recv(2))
return c
clients = [test_client() for i in xrange(5)]
# very next client should not get anything
x = eventlet.with_timeout(0.01,
test_client,
timeout_value="timed out")
self.assertEquals(x, "timed out")
def test_concurrency(self):
evt = event.Event()
def waiter(sock, addr):
sock.sendall(s2b('hi'))
evt.wait()
l = eventlet.listen(('localhost', 0))
gt = eventlet.spawn(eventlet.serve, l, waiter, 5)
def test_client():
c = eventlet.connect(('localhost', l.getsockname()[1]))
# verify the client is connected by getting data
self.assertEquals(s2b('hi'), c.recv(2))
return c
clients = [test_client() for i in xrange(5)]
# very next client should not get anything
x = eventlet.with_timeout(0.01,
test_client,
timeout_value="timed out")
self.assertEquals(x, "timed out")
def test_dispatch_send_recv(ctx):
from eventlet import with_timeout
from eventlet.event import Event
from melkman.messaging import MessageDispatch, always_ack
w = MessageDispatch(ctx)
message_type = "test_dispatch_send_recv"
work_result = Event()
@always_ack
def handler(job, message):
work_result.send(sum(job["values"]))
worker = w.start_worker(message_type, handler)
w.send({"values": [1, 2]}, message_type)
try:
assert with_timeout(2, work_result.wait) == 3
finally:
worker.kill()
worker.wait()
# verify eventlet.listen() accepts in forked children
__test__ = False
if __name__ == '__main__':
import os
import sys
import eventlet
server = eventlet.listen(('127.0.0.1', 0))
result = eventlet.with_timeout(0.01, server.accept, timeout_value=True)
assert result is True, 'Expected timeout'
pid = os.fork()
if pid < 0:
print('fork error')
sys.exit(1)
elif pid == 0:
with eventlet.Timeout(1):
sock, _ = server.accept()
sock.sendall('ok {0}'.format(os.getpid()).encode())
sock.close()
sys.exit(0)
elif pid > 0:
with eventlet.Timeout(1):
sock = eventlet.connect(server.getsockname())
data = sock.recv(20).decode()
assert data.startswith('ok ')
spid = int(data[3:].strip())
assert spid == pid
kpid, status = os.wait()
assert kpid == pid
def serve_from_child(sock, config, controller_pid):
threads = config.get('threadpool_workers', 0)
wsgi_application = spawning.util.named(config['app_factory'])(config)
if config.get('coverage'):
wsgi_application = FigleafCoverage(wsgi_application)
if threads > 1:
wsgi_application = ExecuteInThreadpool(wsgi_application)
elif threads != 1:
print "(%s) not using threads, installing eventlet cooperation monkeypatching" % (
os.getpid(), )
eventlet.patcher.monkey_patch(all=False, socket=True)
host, port = sock.getsockname()
access_log_file = config.get('access_log_file')
if access_log_file is not None:
access_log_file = open(access_log_file, 'a')
max_age = 0
if config.get('max_age'):
max_age = int(config.get('max_age'))
server_event = eventlet.event.Event()
http_version = config.get('no_keepalive') and 'HTTP/1.0' or 'HTTP/1.1'
try:
wsgi_args = (sock, wsgi_application)
wsgi_kwargs = {'log' : access_log_file, 'server_event' : server_event, 'max_http_version' : http_version}
if config.get('no_keepalive'):
wsgi_kwargs.update({'keepalive' : False})
if max_age:
wsgi_kwargs.update({'timeout_value' : True})
eventlet.with_timeout(max_age, eventlet.wsgi.server, *wsgi_args,
**wsgi_kwargs)
else:
eventlet.wsgi.server(*wsgi_args, **wsgi_kwargs)
except KeyboardInterrupt:
pass
except ExitChild:
pass
## Set a deadman timer to violently kill the process if it doesn't die after
## some long timeout.
signal.signal(signal.SIGALRM, deadman_timeout)
signal.alarm(config['deadman_timeout'])
## Once we get here, we just need to handle outstanding sockets, not
## accept any new sockets, so we should close the server socket.
sock.close()
server = server_event.wait()
last_outstanding = None
if server.outstanding_requests:
## Let's tell our parent that we're dying
try:
os.kill(controller_pid, signal.SIGUSR1)
except OSError, e:
if not e.errno == errno.ESRCH:
raise
def serve_from_child(sock, config, controller_pid):
threads = config.get('threadpool_workers', 0)
wsgi_application = spawning.util.named(config['app_factory'])(config)
if config.get('coverage'):
wsgi_application = FigleafCoverage(wsgi_application)
if config.get('sysinfo'):
wsgi_application = SystemInfo(wsgi_application)
if threads >= 1:
# proxy calls of the application through tpool
wsgi_application = tpool_wsgi(wsgi_application)
elif threads != 1:
print "(%s) not using threads, installing eventlet cooperation monkeypatching" % (
os.getpid(), )
eventlet.patcher.monkey_patch(all=False, socket=True)
host, port = sock.getsockname()
access_log_file = config.get('access_log_file')
if access_log_file is not None:
access_log_file = open(access_log_file, 'a')
max_age = 0
if config.get('max_age'):
max_age = int(config.get('max_age'))
server_event = eventlet.event.Event()
# the status object wants to have a reference to the server object
if config.get('status_port'):
def send_server_to_status(server_event):
server = server_event.wait()
get_statusobj().server = server
eventlet.spawn(send_server_to_status, server_event)
http_version = config.get('no_keepalive') and 'HTTP/1.0' or 'HTTP/1.1'
try:
wsgi_args = (sock, wsgi_application)
wsgi_kwargs = {'log' : access_log_file, 'server_event' : server_event, 'max_http_version' : http_version}
if config.get('no_keepalive'):
wsgi_kwargs.update({'keepalive' : False})
if max_age:
wsgi_kwargs.update({'timeout_value' : True})
eventlet.with_timeout(max_age, eventlet.wsgi.server, *wsgi_args,
**wsgi_kwargs)
warn_controller_of_imminent_death(controller_pid)
else:
eventlet.wsgi.server(*wsgi_args, **wsgi_kwargs)
except KeyboardInterrupt:
# controller probably doesn't know that we got killed by a SIGINT
warn_controller_of_imminent_death(controller_pid)
except ExitChild:
pass # parent killed us, it already knows we're dying
## Set a deadman timer to violently kill the process if it doesn't die after
## some long timeout.
signal.signal(signal.SIGALRM, deadman_timeout)
signal.alarm(config['deadman_timeout'])
## Once we get here, we just need to handle outstanding sockets, not
## accept any new sockets, so we should close the server socket.
sock.close()
server = server_event.wait()
last_outstanding = None
while server.outstanding_requests:
if last_outstanding != server.outstanding_requests:
print "(%s) %s requests remaining, waiting... (timeout after %s)" % (
os.getpid(), server.outstanding_requests, config['deadman_timeout'])
last_outstanding = server.outstanding_requests
eventlet.sleep(0.1)
print "(%s) *** Child exiting: all requests completed at %s" % (
os.getpid(), time.asctime())
def serve_from_child(sock, config, controller_pid):
threads = config.get("threadpool_workers", 0)
wsgi_application = spawning.util.named(config["app_factory"])(config)
if config.get("coverage"):
wsgi_application = FigleafCoverage(wsgi_application)
if threads > 1:
from eventlet import tpool
# proxy calls of the application through tpool, and ensure that
# iteration over the iterator goes through the pool as well
wsgi_application = tpool.Proxy(wsgi_application, autowrap_names=("__call__",))
elif threads != 1:
print "(%s) not using threads, installing eventlet cooperation monkeypatching" % (os.getpid(),)
eventlet.patcher.monkey_patch(all=False, socket=True)
host, port = sock.getsockname()
access_log_file = config.get("access_log_file")
if access_log_file is not None:
access_log_file = open(access_log_file, "a")
max_age = 0
if config.get("max_age"):
max_age = int(config.get("max_age"))
server_event = eventlet.event.Event()
http_version = config.get("no_keepalive") and "HTTP/1.0" or "HTTP/1.1"
try:
wsgi_args = (sock, wsgi_application)
wsgi_kwargs = {"log": access_log_file, "server_event": server_event, "max_http_version": http_version}
if config.get("no_keepalive"):
wsgi_kwargs.update({"keepalive": False})
if max_age:
wsgi_kwargs.update({"timeout_value": True})
eventlet.with_timeout(max_age, eventlet.wsgi.server, *wsgi_args, **wsgi_kwargs)
else:
eventlet.wsgi.server(*wsgi_args, **wsgi_kwargs)
except KeyboardInterrupt:
pass
except ExitChild:
pass
## Set a deadman timer to violently kill the process if it doesn't die after
## some long timeout.
signal.signal(signal.SIGALRM, deadman_timeout)
signal.alarm(config["deadman_timeout"])
## Once we get here, we just need to handle outstanding sockets, not
## accept any new sockets, so we should close the server socket.
sock.close()
server = server_event.wait()
last_outstanding = None
if server.outstanding_requests:
## Let's tell our parent that we're dying
try:
os.kill(controller_pid, signal.SIGUSR1)
except OSError, e:
if not e.errno == errno.ESRCH:
raise
def serve_from_child(sock, config, controller_pid):
threads = config.get('threadpool_workers', 0)
wsgi_application = spawning.util.named(config['app_factory'])(config)
if config.get('coverage'):
wsgi_application = FigleafCoverage(wsgi_application)
if config.get('sysinfo'):
wsgi_application = SystemInfo(wsgi_application)
if threads >= 1:
# proxy calls of the application through tpool
wsgi_application = tpool_wsgi(wsgi_application)
elif threads != 1:
from eventlet.green import socket
host, port = sock.getsockname()
access_log_file = config.get('access_log_file')
if access_log_file is not None:
access_log_file = open(access_log_file, 'a')
max_age = 0
if config.get('max_age'):
max_age = int(config.get('max_age'))
server_event = eventlet.event.Event()
# the status object wants to have a reference to the server object
if config.get('status_port'):
def send_server_to_status(server_event):
server = server_event.wait()
get_statusobj().server = server
eventlet.spawn(send_server_to_status, server_event)
http_version = config.get('no_keepalive') and 'HTTP/1.0' or 'HTTP/1.1'
try:
wsgi_args = (sock, wsgi_application)
wsgi_kwargs = {
'log': access_log_file,
'server_event': server_event,
'max_http_version': http_version
}
if config.get('no_keepalive'):
wsgi_kwargs.update({'keepalive': False})
if max_age:
wsgi_kwargs.update({'timeout_value': True})
eventlet.with_timeout(max_age, eventlet.wsgi.server, *wsgi_args,
**wsgi_kwargs)
warn_controller_of_imminent_death(controller_pid)
else:
eventlet.wsgi.server(*wsgi_args, **wsgi_kwargs)
except KeyboardInterrupt:
# controller probably doesn't know that we got killed by a SIGINT
warn_controller_of_imminent_death(controller_pid)
except ExitChild:
pass # parent killed us, it already knows we're dying
## Once we get here, we should not accept any new sockets, so we should close the server socket.
sock.close()
server = server_event.wait()
print "(%s) *** Child exiting" % (os.getpid(), )
os.kill(os.getpid(), signal.SIGKILL)
def test_pauseresume_producing(self):
self.conn.pauseProducing()
spawn_after(DELAY*5, self.conn.resumeProducing)
self.conn.write('hi\r\n')
result = with_timeout(DELAY*10, self.conn.read, timeout_value='timed out')
self.assertEqual('you said hi. BYE', result)
def test_pause_producing(self):
self.conn.pauseProducing()
self.conn.write('hi\r\n')
result = with_timeout(DELAY*10, self.conn.read, timeout_value='timed out')
self.assertEqual('timed out', result)
def longer_timeout():
# this should not catch the outer timeout's exception
return eventlet.with_timeout(DELAY * 10,
eventlet.sleep,
DELAY * 20,
timeout_value='b')
def longer_timeout():
# this should not catch the outer timeout's exception
return eventlet.with_timeout(DELAY * 10, eventlet.sleep, DELAY * 20, timeout_value='b')
if robot_check_result is None:
raise Stop()
except CrawlError, e:
report["result"] = unicode(e)
return report
if robot_check_result == True:
pass
elif robot_check_result == False:
report["result"] = u"Deny by robots.txt"
return report
else:
assert False, u"This branch should not be executed."
report["result"] = u"FIXME: unhandled branch in _process."
return report
fetch_result = with_timeout(settings.socket_timeout, self.io_worker.fetch, uri, timeout_value="timeout")
if fetch_result is None:
raise Stop()
if fetch_result == "timeout":
fetch_result = {}
report["result"] = u"Fetch timeout"
fetch_result.pop("cached", None)
fetch_result.pop("success", None)
total_end_time = time.time()
report["total_time"] = int((total_end_time - total_start_time) * 1000)
report.update(fetch_result)
return report
def func():
return eventlet.with_timeout(0.2, eventlet.sleep, 2, timeout_value=1)
def check_timed_out(self, event, myproc, proc_finished_flag, queue):
X = object()
assert with_timeout(DELAY, event.wait, timeout_value=X) is X
assert with_timeout(DELAY, queue.wait, timeout_value=X) is X
assert with_timeout(DELAY, proc.waitall, [myproc], timeout_value=X) is X
assert proc_finished_flag == [], proc_finished_flag
def serve_from_child(sock, config, controller_pid):
threads = config.get('threadpool_workers', 0)
wsgi_application = spawning.util.named(config['app_factory'])(config)
if config.get('coverage'):
wsgi_application = FigleafCoverage(wsgi_application)
if config.get('sysinfo'):
wsgi_application = SystemInfo(wsgi_application)
if threads >= 1:
# proxy calls of the application through tpool
wsgi_application = tpool_wsgi(wsgi_application)
elif threads != 1:
print "(%s) not using threads, installing eventlet cooperation monkeypatching" % (
os.getpid(), )
eventlet.patcher.monkey_patch(all=False, socket=True)
host, port = sock.getsockname()
access_log_file = config.get('access_log_file')
if access_log_file is not None:
access_log_file = open(access_log_file, 'a')
max_age = 0
if config.get('max_age'):
max_age = int(config.get('max_age'))
server_event = eventlet.event.Event()
# the status object wants to have a reference to the server object
if config.get('status_port'):
def send_server_to_status(server_event):
server = server_event.wait()
get_statusobj().server = server
eventlet.spawn(send_server_to_status, server_event)
http_version = config.get('no_keepalive') and 'HTTP/1.0' or 'HTTP/1.1'
try:
wsgi_args = (sock, wsgi_application)
wsgi_kwargs = {
'log': access_log_file,
'server_event': server_event,
'max_http_version': http_version
}
if config.get('no_keepalive'):
wsgi_kwargs.update({'keepalive': False})
if max_age:
wsgi_kwargs.update({'timeout_value': True})
eventlet.with_timeout(max_age, eventlet.wsgi.server, *wsgi_args,
**wsgi_kwargs)
warn_controller_of_imminent_death(controller_pid)
else:
eventlet.wsgi.server(*wsgi_args, **wsgi_kwargs)
except KeyboardInterrupt:
# controller probably doesn't know that we got killed by a SIGINT
warn_controller_of_imminent_death(controller_pid)
except ExitChild:
pass # parent killed us, it already knows we're dying
## Set a deadman timer to violently kill the process if it doesn't die after
## some long timeout.
signal.signal(signal.SIGALRM, deadman_timeout)
signal.alarm(config['deadman_timeout'])
## Once we get here, we just need to handle outstanding sockets, not
## accept any new sockets, so we should close the server socket.
sock.close()
server = server_event.wait()
last_outstanding = None
while server.outstanding_requests:
if last_outstanding != server.outstanding_requests:
print "(%s) %s requests remaining, waiting... (timeout after %s)" % (
os.getpid(), server.outstanding_requests,
config['deadman_timeout'])
last_outstanding = server.outstanding_requests
eventlet.sleep(0.1)
print "(%s) *** Child exiting: all requests completed at %s" % (
os.getpid(), time.asctime())
def func():
return eventlet.with_timeout(0.2,
eventlet.sleep,
2,
timeout_value=1)
