def __init__(
self,
handler,
opts=None,
plugins=(),
# A calibre logging object. If None, a default log that logs to
# stdout is used
log=None,
# A calibre logging object for access logging, by default no access
# logging is performed
access_log=None):
self.ready = False
self.handler = handler
self.opts = opts or Options()
if self.opts.trusted_ips:
self.opts.trusted_ips = tuple(
parse_trusted_ips(self.opts.trusted_ips))
self.log = log or ThreadSafeLog(level=ThreadSafeLog.DEBUG)
self.jobs_manager = JobsManager(self.opts, self.log)
self.access_log = access_log
ba = (self.opts.listen_on, int(self.opts.port))
if not ba[0]:
# AI_PASSIVE does not work with host of '' or None
ba = ('0.0.0.0', ba[1])
self.bind_address = ba
self.bound_address = None
self.connection_map = {}
self.ssl_context = None
if self.opts.ssl_certfile is not None and self.opts.ssl_keyfile is not None:
self.ssl_context = ssl.create_default_context(
ssl.Purpose.CLIENT_AUTH)
self.ssl_context.load_cert_chain(certfile=self.opts.ssl_certfile,
keyfile=self.opts.ssl_keyfile)
self.ssl_context.set_servername_callback(self.on_ssl_servername)
self.pre_activated_socket = None
if self.opts.allow_socket_preallocation:
from calibre.srv.pre_activated import pre_activated_socket
self.pre_activated_socket = pre_activated_socket()
if self.pre_activated_socket is not None:
set_socket_inherit(self.pre_activated_socket, False)
self.bind_address = self.pre_activated_socket.getsockname()
self.create_control_connection()
self.pool = ThreadPool(self.log,
self.job_completed,
count=self.opts.worker_count)
self.plugin_pool = PluginPool(self, plugins)
def __init__(
self,
handler,
opts=None,
plugins=(),
# A calibre logging object. If None, a default log that logs to
# stdout is used
log=None,
# A calibre logging object for access logging, by default no access
# logging is performed
access_log=None
):
self.ready = False
self.handler = handler
self.opts = opts or Options()
self.log = log or ThreadSafeLog(level=ThreadSafeLog.DEBUG)
self.jobs_manager = JobsManager(self.opts, self.log)
self.access_log = access_log
ba = (self.opts.listen_on, int(self.opts.port))
if not ba[0]:
# AI_PASSIVE does not work with host of '' or None
ba = ('0.0.0.0', ba[1])
self.bind_address = ba
self.bound_address = None
self.connection_map = {}
self.ssl_context = None
if self.opts.ssl_certfile is not None and self.opts.ssl_keyfile is not None:
self.ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self.ssl_context.load_cert_chain(certfile=self.opts.ssl_certfile, keyfile=self.opts.ssl_keyfile)
self.ssl_context.set_servername_callback(self.on_ssl_servername)
self.pre_activated_socket = None
if self.opts.allow_socket_preallocation:
from calibre.srv.pre_activated import pre_activated_socket
self.pre_activated_socket = pre_activated_socket()
if self.pre_activated_socket is not None:
set_socket_inherit(self.pre_activated_socket, False)
self.bind_address = self.pre_activated_socket.getsockname()
self.create_control_connection()
self.pool = ThreadPool(self.log, self.job_completed, count=self.opts.worker_count)
self.plugin_pool = PluginPool(self, plugins)
def test_jobs_manager(self):
'Test the jobs manager'
from calibre.srv.jobs import JobsManager
O = namedtuple('O', 'max_jobs max_job_time')
class FakeLog(list):
def error(self, *args):
self.append(' '.join(args))
jm = JobsManager(O(1, 5), FakeLog())
job_id = jm.start_job('simple test',
'calibre.srv.jobs',
'sleep_test',
args=(1.0, ))
job_id2 = jm.start_job('t2',
'calibre.srv.jobs',
'sleep_test',
args=(3, ))
jid = jm.start_job('err test', 'calibre.srv.jobs', 'error_test')
status = jm.job_status(job_id)[0]
s = ('waiting', 'running')
self.assertIn(status, s)
status2 = jm.job_status(job_id2)[0]
self.assertEqual(status2, 'waiting')
while jm.job_status(job_id)[0] in s:
time.sleep(0.01)
status, result, tb, was_aborted = jm.job_status(job_id)
self.assertEqual(status, 'finished')
self.assertFalse(was_aborted)
self.assertFalse(tb)
self.assertEqual(result, 1.0)
status2 = jm.job_status(job_id2)[0]
time.sleep(0.01)
self.assertEqual(status2, 'running')
jm.abort_job(job_id2)
self.assertTrue(jm.wait_for_running_job(job_id2))
status, result, tb, was_aborted = jm.job_status(job_id2)
self.assertTrue(was_aborted)
self.assertTrue(jm.wait_for_running_job(jid))
status, result, tb, was_aborted = jm.job_status(jid)
self.assertTrue(tb), self.assertIn('a testing error', tb)
jm.start_job('simple test',
'calibre.srv.jobs',
'sleep_test',
args=(1.0, ))
jm.shutdown(), jm.wait_for_shutdown(monotonic() + 1)
class ServerLoop(object):
LISTENING_MSG = 'calibre server listening on'
def __init__(
self,
handler,
opts=None,
plugins=(),
# A calibre logging object. If None, a default log that logs to
# stdout is used
log=None,
# A calibre logging object for access logging, by default no access
# logging is performed
access_log=None):
self.ready = False
self.handler = handler
self.opts = opts or Options()
self.log = log or ThreadSafeLog(level=ThreadSafeLog.DEBUG)
self.jobs_manager = JobsManager(self.opts, self.log)
self.access_log = access_log
ba = (self.opts.listen_on, int(self.opts.port))
if not ba[0]:
# AI_PASSIVE does not work with host of '' or None
ba = ('0.0.0.0', ba[1])
self.bind_address = ba
self.bound_address = None
self.connection_map = {}
self.ssl_context = None
if self.opts.ssl_certfile is not None and self.opts.ssl_keyfile is not None:
self.ssl_context = ssl.create_default_context(
ssl.Purpose.CLIENT_AUTH)
self.ssl_context.load_cert_chain(certfile=self.opts.ssl_certfile,
keyfile=self.opts.ssl_keyfile)
self.ssl_context.set_servername_callback(self.on_ssl_servername)
self.pre_activated_socket = None
if self.opts.allow_socket_preallocation:
from calibre.srv.pre_activated import pre_activated_socket
self.pre_activated_socket = pre_activated_socket()
if self.pre_activated_socket is not None:
set_socket_inherit(self.pre_activated_socket, False)
self.bind_address = self.pre_activated_socket.getsockname()
self.create_control_connection()
self.pool = ThreadPool(self.log,
self.job_completed,
count=self.opts.worker_count)
self.plugin_pool = PluginPool(self, plugins)
def on_ssl_servername(self, socket, server_name, ssl_context):
c = self.connection_map.get(socket.fileno())
if getattr(c, 'ssl_handshake_done', False):
c.ready = False
c.ssl_terminated = True
# We do not allow client initiated SSL renegotiation
return ssl.ALERT_DESCRIPTION_NO_RENEGOTIATION
def create_control_connection(self):
self.control_in, self.control_out = create_sock_pair()
def __str__(self):
return "%s(%r)" % (self.__class__.__name__, self.bind_address)
__repr__ = __str__
@property
def num_active_connections(self):
return len(self.connection_map)
def do_bind(self):
# Get the correct address family for our host (allows IPv6 addresses)
host, port = self.bind_address
try:
info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
except socket.gaierror:
if ':' in host:
info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "",
self.bind_address + (0, 0))]
else:
info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "",
self.bind_address)]
self.socket = None
msg = "No socket could be created"
for res in info:
af, socktype, proto, canonname, sa = res
try:
self.bind(af, socktype, proto)
except socket.error as serr:
msg = "%s -- (%s: %s)" % (msg, sa, as_unicode(serr))
if self.socket:
self.socket.close()
self.socket = None
continue
break
if not self.socket:
raise socket.error(msg)
def initialize_socket(self):
if self.pre_activated_socket is None:
try:
self.do_bind()
except socket.error as err:
if not self.opts.fallback_to_detected_interface:
raise
ip = get_external_ip()
if ip == self.bind_address[0]:
raise
self.log.warn(
'Failed to bind to %s with error: %s. Trying to bind to the default interface: %s instead'
% (self.bind_address[0], as_unicode(err), ip))
self.bind_address = (ip, self.bind_address[1])
self.do_bind()
else:
self.socket = self.pre_activated_socket
self.pre_activated_socket = None
self.setup_socket()
def serve(self):
self.connection_map = {}
self.socket.listen(min(socket.SOMAXCONN, 128))
self.bound_address = ba = self.socket.getsockname()
if isinstance(ba, tuple):
ba = ':'.join(map(type(''), ba))
self.pool.start()
with TemporaryDirectory(prefix='srv-') as tdir:
self.tdir = tdir
self.ready = True
if self.LISTENING_MSG:
self.log(self.LISTENING_MSG, ba)
self.plugin_pool.start()
while self.ready:
try:
self.tick()
except SystemExit:
self.shutdown()
raise
except KeyboardInterrupt:
break
except:
self.log.exception('Error in ServerLoop.tick')
self.shutdown()
def serve_forever(self):
""" Listen for incoming connections. """
self.initialize_socket()
self.serve()
def setup_socket(self):
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# If listening on the IPV6 any address ('::' = IN6ADDR_ANY),
# activate dual-stack.
if (hasattr(socket, 'AF_INET6')
and self.socket.family == socket.AF_INET6
and self.bind_address[0] in ('::', '::0', '::0.0.0.0')):
try:
self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY,
0)
except (AttributeError, socket.error):
# Apparently, the socket option is not available in
# this machine's TCP stack
pass
self.socket.setblocking(0)
def bind(self, family, atype, proto=0):
'''Create (or recreate) the actual socket object.'''
self.socket = socket.socket(family, atype, proto)
set_socket_inherit(self.socket, False)
self.setup_socket()
self.socket.bind(self.bind_address)
def tick(self):
now = monotonic()
read_needed, write_needed, readable, remove, close_needed = [], [], [], [], []
has_ssl = self.ssl_context is not None
for s, conn in self.connection_map.iteritems():
if now - conn.last_activity > self.opts.timeout:
if conn.handle_timeout():
conn.last_activity = now
else:
remove.append((s, conn))
continue
wf = conn.wait_for
if wf is READ or wf is RDWR:
if wf is RDWR:
write_needed.append(s)
if conn.read_buffer.has_data:
readable.append(s)
else:
if has_ssl:
conn.drain_ssl_buffer()
if conn.ready:
(readable if conn.read_buffer.has_data else
read_needed).append(s)
else:
close_needed.append((s, conn))
else:
read_needed.append(s)
elif wf is WRITE:
write_needed.append(s)
for s, conn in remove:
self.log('Closing connection because of extended inactivity: %s' %
conn.state_description)
self.close(s, conn)
for x, conn in close_needed:
self.close(s, conn)
if readable:
writable = []
else:
try:
readable, writable, _ = select.select(
[self.socket.fileno(),
self.control_out.fileno()] + read_needed, write_needed,
[], self.opts.timeout)
except ValueError: # self.socket.fileno() == -1
self.ready = False
self.log.error('Listening socket was unexpectedly terminated')
return
except (select.error, socket.error) as e:
# select.error has no errno attribute. errno is instead
# e.args[0]
if getattr(e, 'errno', e.args[0]) in socket_errors_eintr:
return
for s, conn in tuple(self.connection_map.iteritems()):
try:
select.select([s], [], [], 0)
except (select.error, socket.error) as e:
if getattr(e, 'errno',
e.args[0]) not in socket_errors_eintr:
self.close(s, conn) # Bad socket, discard
return
if not self.ready:
return
ignore = set()
for s, conn, event in self.get_actions(readable, writable):
if s in ignore:
continue
try:
conn.handle_event(event)
if not conn.ready:
self.close(s, conn)
except JobQueueFull:
self.log.exception('Server busy handling request: %s' %
conn.state_description)
if conn.ready:
if conn.response_started:
self.close(s, conn)
else:
try:
conn.report_busy()
except Exception:
self.close(s, conn)
except Exception as e:
ignore.add(s)
ssl_terminated = getattr(conn, 'ssl_terminated', False)
if ssl_terminated:
self.log.warn(
'Client tried to initiate SSL renegotiation, closing connection'
)
self.close(s, conn)
else:
self.log.exception('Unhandled exception in state: %s' %
conn.state_description)
if conn.ready:
if conn.response_started:
self.close(s, conn)
else:
try:
conn.report_unhandled_exception(
e, traceback.format_exc())
except Exception:
self.close(s, conn)
else:
self.log.error(
'Error in SSL handshake, terminating connection: %s'
% as_unicode(e))
self.close(s, conn)
def wakeup(self):
self.control_in.sendall(WAKEUP)
def job_completed(self):
self.control_in.sendall(JOB_DONE)
def dispatch_job_results(self):
while True:
try:
s, ok, result = self.pool.get_nowait()
except Empty:
break
conn = self.connection_map.get(s)
if conn is not None:
yield s, conn, (ok, result)
def close(self, s, conn):
self.connection_map.pop(s, None)
conn.close()
def get_actions(self, readable, writable):
listener = self.socket.fileno()
control = self.control_out.fileno()
for s in readable:
if s == listener:
sock, addr = self.accept()
if sock is not None:
s = sock.fileno()
if s > -1:
self.connection_map[s] = conn = self.handler(
sock, self.opts, self.ssl_context, self.tdir, addr,
self.pool, self.log, self.access_log, self.wakeup)
if self.ssl_context is not None:
yield s, conn, RDWR
elif s == control:
try:
c = self.control_out.recv(1)
except socket.error:
if not self.ready:
return
self.log.error('Control socket raised an error, resetting')
self.create_control_connection()
continue
if c == JOB_DONE:
for s, conn, event in self.dispatch_job_results():
yield s, conn, event
elif c == WAKEUP:
pass
elif not c:
if not self.ready:
return
self.log.error(
'Control socket failed to recv(), resetting')
self.create_control_connection()
else:
yield s, self.connection_map[s], READ
for s in writable:
try:
conn = self.connection_map[s]
except KeyError:
continue # Happens if connection was closed during read phase
yield s, conn, WRITE
def accept(self):
try:
sock, addr = self.socket.accept()
set_socket_inherit(sock, False), sock.setblocking(False)
return sock, addr
except socket.error:
return None, None
def stop(self):
self.ready = False
self.wakeup()
def shutdown(self):
self.jobs_manager.shutdown()
try:
if getattr(self, 'socket', None):
self.socket.close()
self.socket = None
except socket.error:
pass
for s, conn in tuple(self.connection_map.iteritems()):
self.close(s, conn)
wait_till = monotonic() + self.opts.shutdown_timeout
for pool in (self.plugin_pool, self.pool):
pool.stop(wait_till)
if pool.workers:
self.log.warn('Failed to shutdown %d workers in %s cleanly' %
(len(pool.workers), pool.__class__.__name__))
self.jobs_manager.wait_for_shutdown(wait_till)
def test_jobs_manager(self):
'Test the jobs manager'
from calibre.srv.jobs import JobsManager
O = namedtuple('O', 'max_jobs max_job_time')
class FakeLog(list):
def error(self, *args):
self.append(' '.join(args))
s = ('waiting', 'running')
jm = JobsManager(O(1, 5), FakeLog())
def job_status(jid):
return jm.job_status(jid)[0]
# Start jobs
job_id1 = jm.start_job('simple test',
'calibre.srv.jobs',
'sleep_test',
args=(1.0, ))
job_id2 = jm.start_job('t2',
'calibre.srv.jobs',
'sleep_test',
args=(3, ))
job_id3 = jm.start_job('err test', 'calibre.srv.jobs', 'error_test')
# Job 1
job_id = job_id1
status = jm.job_status(job_id)[0]
self.assertIn(status, s)
for jid in (job_id2, job_id3):
self.assertEqual(job_status(jid), 'waiting')
while job_status(job_id) in s:
time.sleep(0.01)
status, result, tb, was_aborted = jm.job_status(job_id)
self.assertEqual(status, 'finished')
self.assertFalse(was_aborted)
self.assertFalse(tb)
self.assertEqual(result, 1.0)
# Job 2
job_id = job_id2
while job_status(job_id) == 'waiting':
time.sleep(0.01)
self.assertEqual('running', job_status(job_id))
jm.abort_job(job_id)
self.assertIn(jm.wait_for_running_job(job_id), (True, None))
status, result, tb, was_aborted = jm.job_status(job_id)
self.assertEqual('finished', status)
self.assertTrue(was_aborted)
# Job 3
job_id = job_id3
while job_status(job_id) == 'waiting':
time.sleep(0.01)
self.assertIn(jm.wait_for_running_job(job_id), (True, None))
status, result, tb, was_aborted = jm.job_status(job_id)
self.assertEqual(status, 'finished')
self.assertFalse(was_aborted)
self.assertTrue(tb)
self.assertIn('a testing error', tb)
jm.start_job('simple test',
'calibre.srv.jobs',
'sleep_test',
args=(1.0, ))
jm.shutdown(), jm.wait_for_shutdown(monotonic() + 1)
class ServerLoop(object):
LISTENING_MSG = 'calibre server listening on'
def __init__(
self,
handler,
opts=None,
plugins=(),
# A calibre logging object. If None, a default log that logs to
# stdout is used
log=None,
# A calibre logging object for access logging, by default no access
# logging is performed
access_log=None
):
self.ready = False
self.handler = handler
self.opts = opts or Options()
self.log = log or ThreadSafeLog(level=ThreadSafeLog.DEBUG)
self.jobs_manager = JobsManager(self.opts, self.log)
self.access_log = access_log
ba = (self.opts.listen_on, int(self.opts.port))
if not ba[0]:
# AI_PASSIVE does not work with host of '' or None
ba = ('0.0.0.0', ba[1])
self.bind_address = ba
self.bound_address = None
self.connection_map = {}
self.ssl_context = None
if self.opts.ssl_certfile is not None and self.opts.ssl_keyfile is not None:
self.ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self.ssl_context.load_cert_chain(certfile=self.opts.ssl_certfile, keyfile=self.opts.ssl_keyfile)
self.pre_activated_socket = None
if self.opts.allow_socket_preallocation:
from calibre.srv.pre_activated import pre_activated_socket
self.pre_activated_socket = pre_activated_socket()
if self.pre_activated_socket is not None:
set_socket_inherit(self.pre_activated_socket, False)
self.bind_address = self.pre_activated_socket.getsockname()
self.create_control_connection()
self.pool = ThreadPool(self.log, self.job_completed, count=self.opts.worker_count)
self.plugin_pool = PluginPool(self, plugins)
def create_control_connection(self):
self.control_in, self.control_out = create_sock_pair()
def __str__(self):
return "%s(%r)" % (self.__class__.__name__, self.bind_address)
__repr__ = __str__
@property
def num_active_connections(self):
return len(self.connection_map)
def do_bind(self):
# Get the correct address family for our host (allows IPv6 addresses)
host, port = self.bind_address
try:
info = socket.getaddrinfo(
host, port, socket.AF_UNSPEC,
socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
except socket.gaierror:
if ':' in host:
info = [(socket.AF_INET6, socket.SOCK_STREAM,
0, "", self.bind_address + (0, 0))]
else:
info = [(socket.AF_INET, socket.SOCK_STREAM,
0, "", self.bind_address)]
self.socket = None
msg = "No socket could be created"
for res in info:
af, socktype, proto, canonname, sa = res
try:
self.bind(af, socktype, proto)
except socket.error as serr:
msg = "%s -- (%s: %s)" % (msg, sa, as_unicode(serr))
if self.socket:
self.socket.close()
self.socket = None
continue
break
if not self.socket:
raise socket.error(msg)
def serve_forever(self):
""" Listen for incoming connections. """
if self.pre_activated_socket is None:
try:
self.do_bind()
except socket.error as err:
if not self.opts.fallback_to_detected_interface:
raise
ip = get_external_ip()
if ip == self.bind_address[0]:
raise
self.log.warn('Failed to bind to %s with error: %s. Trying to bind to the default interface: %s instead' % (
self.bind_address[0], as_unicode(err), ip))
self.bind_address = (ip, self.bind_address[1])
self.do_bind()
else:
self.socket = self.pre_activated_socket
self.pre_activated_socket = None
self.setup_socket()
self.connection_map = {}
self.socket.listen(min(socket.SOMAXCONN, 128))
self.bound_address = ba = self.socket.getsockname()
if isinstance(ba, tuple):
ba = ':'.join(map(type(''), ba))
self.pool.start()
with TemporaryDirectory(prefix='srv-') as tdir:
self.tdir = tdir
self.ready = True
if self.LISTENING_MSG:
self.log(self.LISTENING_MSG, ba)
self.plugin_pool.start()
while self.ready:
try:
self.tick()
except SystemExit:
self.shutdown()
raise
except KeyboardInterrupt:
break
except:
self.log.exception('Error in ServerLoop.tick')
self.shutdown()
def setup_socket(self):
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# If listening on the IPV6 any address ('::' = IN6ADDR_ANY),
# activate dual-stack.
if (hasattr(socket, 'AF_INET6') and self.socket.family == socket.AF_INET6 and
self.bind_address[0] in ('::', '::0', '::0.0.0.0')):
try:
self.socket.setsockopt(
socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
except (AttributeError, socket.error):
# Apparently, the socket option is not available in
# this machine's TCP stack
pass
self.socket.setblocking(0)
def bind(self, family, atype, proto=0):
'''Create (or recreate) the actual socket object.'''
self.socket = socket.socket(family, atype, proto)
set_socket_inherit(self.socket, False)
self.setup_socket()
self.socket.bind(self.bind_address)
def tick(self):
now = monotonic()
read_needed, write_needed, readable, remove = [], [], [], []
for s, conn in self.connection_map.iteritems():
if now - conn.last_activity > self.opts.timeout:
if conn.handle_timeout():
conn.last_activity = now
else:
remove.append((s, conn))
continue
wf = conn.wait_for
if wf is READ:
(readable if conn.read_buffer.has_data else read_needed).append(s)
elif wf is WRITE:
write_needed.append(s)
elif wf is RDWR:
write_needed.append(s)
(readable if conn.read_buffer.has_data else read_needed).append(s)
for s, conn in remove:
self.log('Closing connection because of extended inactivity: %s' % conn.state_description)
self.close(s, conn)
if readable:
writable = []
else:
try:
readable, writable, _ = select.select([self.socket.fileno(), self.control_out.fileno()] + read_needed, write_needed, [], self.opts.timeout)
except ValueError: # self.socket.fileno() == -1
self.ready = False
self.log.error('Listening socket was unexpectedly terminated')
return
except (select.error, socket.error) as e:
# select.error has no errno attribute. errno is instead
# e.args[0]
if getattr(e, 'errno', e.args[0]) in socket_errors_eintr:
return
for s, conn in tuple(self.connection_map.iteritems()):
try:
select.select([s], [], [], 0)
except (select.error, socket.error) as e:
if getattr(e, 'errno', e.args[0]) not in socket_errors_eintr:
self.close(s, conn) # Bad socket, discard
return
if not self.ready:
return
ignore = set()
for s, conn, event in self.get_actions(readable, writable):
if s in ignore:
continue
try:
conn.handle_event(event)
if not conn.ready:
self.close(s, conn)
except JobQueueFull:
self.log.exception('Server busy handling request: %s' % conn.state_description)
if conn.ready:
if conn.response_started:
self.close(s, conn)
else:
try:
conn.report_busy()
except Exception:
self.close(s, conn)
except Exception as e:
ignore.add(s)
self.log.exception('Unhandled exception in state: %s' % conn.state_description)
if conn.ready:
if conn.response_started:
self.close(s, conn)
else:
try:
conn.report_unhandled_exception(e, traceback.format_exc())
except Exception:
self.close(s, conn)
else:
self.log.error('Error in SSL handshake, terminating connection: %s' % as_unicode(e))
self.close(s, conn)
def wakeup(self):
self.control_in.sendall(WAKEUP)
def job_completed(self):
self.control_in.sendall(JOB_DONE)
def dispatch_job_results(self):
while True:
try:
s, ok, result = self.pool.get_nowait()
except Empty:
break
conn = self.connection_map.get(s)
if conn is not None:
yield s, conn, (ok, result)
def close(self, s, conn):
self.connection_map.pop(s, None)
conn.close()
def get_actions(self, readable, writable):
listener = self.socket.fileno()
control = self.control_out.fileno()
for s in readable:
if s == listener:
sock, addr = self.accept()
if sock is not None:
s = sock.fileno()
if s > -1:
self.connection_map[s] = conn = self.handler(
sock, self.opts, self.ssl_context, self.tdir, addr, self.pool, self.log, self.access_log, self.wakeup)
if self.ssl_context is not None:
yield s, conn, RDWR
elif s == control:
try:
c = self.control_out.recv(1)
except socket.error:
if not self.ready:
return
self.log.error('Control socket raised an error, resetting')
self.create_control_connection()
continue
if c == JOB_DONE:
for s, conn, event in self.dispatch_job_results():
yield s, conn, event
elif c == WAKEUP:
pass
elif not c:
if not self.ready:
return
self.log.error('Control socket failed to recv(), resetting')
self.create_control_connection()
else:
yield s, self.connection_map[s], READ
for s in writable:
try:
conn = self.connection_map[s]
except KeyError:
continue # Happens if connection was closed during read phase
yield s, conn, WRITE
def accept(self):
try:
sock, addr = self.socket.accept()
set_socket_inherit(sock, False), sock.setblocking(False)
return sock, addr
except socket.error:
return None, None
def stop(self):
self.ready = False
self.wakeup()
def shutdown(self):
self.jobs_manager.shutdown()
try:
if getattr(self, 'socket', None):
self.socket.close()
self.socket = None
except socket.error:
pass
for s, conn in tuple(self.connection_map.iteritems()):
self.close(s, conn)
wait_till = monotonic() + self.opts.shutdown_timeout
for pool in (self.plugin_pool, self.pool):
pool.stop(wait_till)
if pool.workers:
self.log.warn('Failed to shutdown %d workers in %s cleanly' % (len(pool.workers), pool.__class__.__name__))
self.jobs_manager.wait_for_shutdown(wait_till)
def test_jobs_manager(self):
'Test the jobs manager'
from calibre.srv.jobs import JobsManager
O = namedtuple('O', 'max_jobs max_job_time')
class FakeLog(list):
def error(self, *args):
self.append(' '.join(args))
s = ('waiting', 'running')
jm = JobsManager(O(1, 5), FakeLog())
def job_status(jid):
return jm.job_status(jid)[0]
# Start jobs
job_id1 = jm.start_job('simple test', 'calibre.srv.jobs', 'sleep_test', args=(1.0,))
job_id2 = jm.start_job('t2', 'calibre.srv.jobs', 'sleep_test', args=(3,))
job_id3 = jm.start_job('err test', 'calibre.srv.jobs', 'error_test')
# Job 1
job_id = job_id1
status = jm.job_status(job_id)[0]
self.assertIn(status, s)
for jid in (job_id2, job_id3):
self.assertEqual(job_status(jid), 'waiting')
while job_status(job_id) in s:
time.sleep(0.01)
status, result, tb, was_aborted = jm.job_status(job_id)
self.assertEqual(status, 'finished')
self.assertFalse(was_aborted)
self.assertFalse(tb)
self.assertEqual(result, 1.0)
# Job 2
job_id = job_id2
while job_status(job_id) == 'waiting':
time.sleep(0.01)
self.assertEqual('running', job_status(job_id))
jm.abort_job(job_id)
self.assertIn(jm.wait_for_running_job(job_id), (True, None))
status, result, tb, was_aborted = jm.job_status(job_id)
self.assertEqual('finished', status)
self.assertTrue(was_aborted)
# Job 3
job_id = job_id3
while job_status(job_id) == 'waiting':
time.sleep(0.01)
self.assertIn(jm.wait_for_running_job(job_id), (True, None))
status, result, tb, was_aborted = jm.job_status(job_id)
self.assertEqual(status, 'finished')
self.assertFalse(was_aborted)
self.assertTrue(tb), self.assertIn('a testing error', tb)
jm.start_job('simple test', 'calibre.srv.jobs', 'sleep_test', args=(1.0,))
jm.shutdown(), jm.wait_for_shutdown(monotonic() + 1)
def test_jobs_manager(self):
'Test the jobs manager'
from calibre.srv.jobs import JobsManager
O = namedtuple('O', 'max_jobs max_job_time')
class FakeLog(list):
def error(self, *args):
self.append(' '.join(args))
jm = JobsManager(O(1, 5), FakeLog())
job_id = jm.start_job('simple test', 'calibre.srv.jobs', 'sleep_test', args=(1.0,))
job_id2 = jm.start_job('t2', 'calibre.srv.jobs', 'sleep_test', args=(3,))
jid = jm.start_job('err test', 'calibre.srv.jobs', 'error_test')
status = jm.job_status(job_id)[0]
s = ('waiting', 'running')
self.assertIn(status, s)
status2 = jm.job_status(job_id2)[0]
self.assertEqual(status2, 'waiting')
while jm.job_status(job_id)[0] in s:
time.sleep(0.01)
status, result, tb, was_aborted = jm.job_status(job_id)
self.assertEqual(status, 'finished')
self.assertFalse(was_aborted)
self.assertFalse(tb)
self.assertEqual(result, 1.0)
status2 = jm.job_status(job_id2)[0]
time.sleep(0.01)
self.assertEqual(status2, 'running')
jm.abort_job(job_id2)
self.assertTrue(jm.wait_for_running_job(job_id2))
status, result, tb, was_aborted = jm.job_status(job_id2)
self.assertTrue(was_aborted)
self.assertTrue(jm.wait_for_running_job(jid))
status, result, tb, was_aborted = jm.job_status(jid)
self.assertTrue(tb), self.assertIn('a testing error', tb)
jm.start_job('simple test', 'calibre.srv.jobs', 'sleep_test', args=(1.0,))
jm.shutdown(), jm.wait_for_shutdown(monotonic() + 1)