def __init__(
self,
name,
worker_threads,
thread_class,
connection,
request_queue=None,
done_queue=None,
sentinel=None,
socket=None,
local=None,
master=None,
):
BaseService.__init__(self, name)
multiprocessing.Process.__init__(self, name=name)
self.worker_threads = worker_threads
self.thread_class = thread_class
self.connection = connection
self.request_queue = request_queue
self.done_queue = done_queue
self.sentinel = sentinel
self.socket = socket
self.is_alive = True
self.master = master
self.local = local
def __init__(self, name, address, worker_threads, threadClass,
requestHandler):
# initialize base service
BaseService.__init__(self, name)
self.addr = address
self.worker_threads = worker_threads
self.requestHandler = requestHandler
self.threadClass = threadClass
# create client request queue
self.requestQueue = Queue.Queue(worker_threads*5)
# create queue for inactive requestHandler objects i.e. those served
self.rhQueue = Queue.Queue(0)
# create threads tuple
self.threadPool = []
def __init__(self, name, address, worker_processes, worker_threads,
thread_class):
# initialize base service
BaseService.__init__(self, name)
self.addr = address
self.worker_threads = worker_threads
self.thread_class = thread_class
self.is_multiprocess = False
if multiprocessing:
if worker_processes == 'auto':
cpus = multiprocessing.cpu_count()
if cpus == 1:
worker_processes = 0
else:
worker_processes = cpus
else:
worker_processes = int(worker_processes)
self.is_multiprocess = worker_processes > 0
self.worker_processes = worker_processes
self.pipes = [] # used for sending management tasks
# to subrpocesses
self.task_dispatchers = [] # used for getting completed requests
# from subprocesses only when sockets are
# not pickleable
self.sentinel = None
self._socket = None
request_queue = None
done_queue = None
if self.is_multiprocess:
if not hasattr(socket, 'fromfd'):
# create queues for communicating
request_queue = get_shared_queue(2048)
done_queue = get_shared_queue(2048)
self.sentinel = (-1, b'EOF')
else:
request_queue = queue.Queue(worker_threads * 2)
Dispatcher.__init__(self, request_queue, done_queue)
# create worker tuple
self.worker_pool = []
def __init__(self, name, address, worker_processes, worker_threads,
thread_class):
# initialize base service
BaseService.__init__(self, name)
self.addr = address
self.worker_threads = worker_threads
self.thread_class = thread_class
self.is_multiprocess = False
if multiprocessing:
if worker_processes == 'auto':
cpus = multiprocessing.cpu_count()
if cpus == 1:
worker_processes = 0
else:
worker_processes = cpus
else:
worker_processes = int(worker_processes)
self.is_multiprocess = worker_processes > 0
self.worker_processes = worker_processes
self.pipes = [] # used for sending management tasks
# to subrpocesses
self.task_dispatchers = [] # used for getting completed requests
# from subprocesses only when sockets are
# not pickleable
self.sentinel = None
self._socket = None
request_queue = None
done_queue = None
if self.is_multiprocess:
if not hasattr(socket, 'fromfd'):
# create queues for communicating
request_queue = get_shared_queue(2048)
done_queue = get_shared_queue(2048)
self.sentinel = (-1, b'EOF')
else:
request_queue = queue.Queue(worker_threads * 2)
Dispatcher.__init__(self, request_queue, done_queue)
# create worker tuple
self.worker_pool = []
def start(self):
# start runtime services
BaseService.start(self)
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setblocking(0)
try:
self._socket.bind(self.addr)
except socket.error as v:
self._socket.close()
raise v
self._socket.listen(64)
if self.request_queue is not None:
# activate server socket
self.set_socket(self._socket)
# start workers
self._start_workers()
self.running = True
def start(self):
# start runtime services
BaseService.start(self)
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setblocking(0)
try:
self._socket.bind(self.addr)
except socket.error as v:
self._socket.close()
raise v
self._socket.listen(64)
if self.request_queue is not None:
# activate server socket
self.set_socket(self._socket)
# start workers
self._start_workers()
self.running = True
def __init__(self,
name,
worker_threads,
thread_class,
connection,
request_queue=None,
done_queue=None,
sentinel=None,
socket=None,
local=None,
master=None):
BaseService.__init__(self, name)
multiprocessing.Process.__init__(self, name=name)
self.worker_threads = worker_threads
self.thread_class = thread_class
self.connection = connection
self.request_queue = request_queue
self.done_queue = done_queue
self.sentinel = sentinel
self.socket = socket
self.is_alive = True
self.master = master
self.local = local
def shutdown(self):
if self.running:
self.running = False
self._stop_workers()
# shut down runtime services
BaseService.shutdown(self)
def unlock_db(self):
BaseService.unlock_db(self)
if self.is_multiprocess:
self.send("DB_UNLOCK")
def lock_db(self):
BaseService.lock_db(self)
if self.is_multiprocess:
self.send("DB_LOCK")
def remove_runtime_service(self, component):
if self.is_multiprocess and component == "db":
self.send("DB_CLOSE")
BaseService.remove_runtime_service(self, component)
def add_runtime_service(self, component, *args, **kwargs):
inited = BaseService.add_runtime_service(self, component, *args, **kwargs)
if self.is_multiprocess and component == "db":
self.send("DB_OPEN")
return inited
def start(self):
BaseService.start(self)
self.thread = Thread(name='%s thread' % self.name,
target=self.thread_loop)
self.running = True
self.thread.start()
def run(self):
# start runtime services
BaseService.start(self)
# set initial site master
if self.master is not None:
from porcupine.db import _db
rep_mgr = _db.get_replication_manager()
rep_mgr.master = self.master
rep_mgr.local_site = self.local
# start server
if self.socket is not None:
socket_map = {}
# start server
self.request_queue = queue.Queue(self.worker_threads * 2)
self.done_queue = None
server = Dispatcher(self.request_queue, None, socket_map)
# activate server socket
server.set_socket(self.socket, socket_map)
# start asyncore loop
asyn_thread = Thread(target=self._async_loop, args=(socket_map,), name="%s asyncore thread" % self.name)
asyn_thread.start()
else:
# create queue for inactive RequestHandlerProxy objects
# i.e. those served
self.rhproxy_queue = queue.Queue(0)
# patch shared queues
self.request_queue = init_queue(*self.request_queue)
self.done_queue = init_queue(*self.done_queue)
thread_pool = []
for i in range(self.worker_threads):
tname = "%s thread %d" % (self.name, i + 1)
t = self.thread_class(target=self._thread_loop, name=tname)
thread_pool.append(t)
# start management thread
mt = Thread(target=self._manage, name="%s management thread" % self.name)
thread_pool.append(mt)
# start threads
[t.start() for t in thread_pool]
try:
while self.is_alive:
time.sleep(8.0)
except KeyboardInterrupt:
pass
except IOError:
pass
if self.socket is not None:
for i in range(self.worker_threads):
self.request_queue.put(self.sentinel)
# join threads
for t in thread_pool:
t.join()
if self.socket is not None:
# join asyncore thread
asyncore.close_all(socket_map)
asyn_thread.join()
# shutdown runtime services
BaseService.shutdown(self)
def shutdown(self):
if self.running:
self.running = False
self._stop_workers()
# shut down runtime services
BaseService.shutdown(self)
def unlock_db(self):
BaseService.unlock_db(self)
if self.is_multiprocess:
self.send('DB_UNLOCK')
def lock_db(self):
BaseService.lock_db(self)
if self.is_multiprocess:
self.send('DB_LOCK')
def remove_runtime_service(self, component):
if self.is_multiprocess and component == 'db':
self.send('DB_CLOSE')
BaseService.remove_runtime_service(self, component)
def add_runtime_service(self, component, *args, **kwargs):
inited = BaseService.add_runtime_service(self, component, *args,
**kwargs)
if self.is_multiprocess and component == 'db':
self.send('DB_OPEN')
return inited
def __init__(self, name, interval):
BaseService.__init__(self, name)
self.interval = interval
def run(self):
# start runtime services
BaseService.start(self)
# set initial site master
if self.master is not None:
from porcupine.db import _db
rep_mgr = _db.get_replication_manager()
rep_mgr.master = self.master
rep_mgr.local_site = self.local
# start server
if self.socket is not None:
socket_map = {}
# start server
self.request_queue = queue.Queue(self.worker_threads * 2)
self.done_queue = None
server = Dispatcher(self.request_queue, None, socket_map)
# activate server socket
server.set_socket(self.socket, socket_map)
# start asyncore loop
asyn_thread = Thread(target=self._async_loop,
args=(socket_map, ),
name='%s asyncore thread' % self.name)
asyn_thread.start()
else:
# create queue for inactive RequestHandlerProxy objects
# i.e. those served
self.rhproxy_queue = queue.Queue(0)
# patch shared queues
self.request_queue = init_queue(*self.request_queue)
self.done_queue = init_queue(*self.done_queue)
thread_pool = []
for i in range(self.worker_threads):
tname = '%s thread %d' % (self.name, i + 1)
t = self.thread_class(target=self._thread_loop, name=tname)
thread_pool.append(t)
# start management thread
mt = Thread(target=self._manage,
name='%s management thread' % self.name)
thread_pool.append(mt)
# start threads
[t.start() for t in thread_pool]
try:
while self.is_alive:
time.sleep(8.0)
except KeyboardInterrupt:
pass
except IOError:
pass
if self.socket is not None:
for i in range(self.worker_threads):
self.request_queue.put(self.sentinel)
# join threads
for t in thread_pool:
t.join()
if self.socket is not None:
# join asyncore thread
asyncore.close_all(socket_map)
asyn_thread.join()
# shutdown runtime services
BaseService.shutdown(self)
def shutdown(self):
self.running = False
self.thread.join()
BaseService.shutdown(self)
def remove_runtime_service(self, component):
if self.is_multiprocess and component == 'db':
self.send('DB_CLOSE')
BaseService.remove_runtime_service(self, component)
