def __init__( self, name, worker_threads, thread_class, connection, request_queue=None, done_queue=None, sentinel=None, socket=None, local=None, master=None, ): BaseService.__init__(self, name) multiprocessing.Process.__init__(self, name=name) self.worker_threads = worker_threads self.thread_class = thread_class self.connection = connection self.request_queue = request_queue self.done_queue = done_queue self.sentinel = sentinel self.socket = socket self.is_alive = True self.master = master self.local = local
def __init__(self, name, address, worker_threads, threadClass, requestHandler): # initialize base service BaseService.__init__(self, name) self.addr = address self.worker_threads = worker_threads self.requestHandler = requestHandler self.threadClass = threadClass # create client request queue self.requestQueue = Queue.Queue(worker_threads*5) # create queue for inactive requestHandler objects i.e. those served self.rhQueue = Queue.Queue(0) # create threads tuple self.threadPool = []
def __init__(self, name, address, worker_processes, worker_threads, thread_class): # initialize base service BaseService.__init__(self, name) self.addr = address self.worker_threads = worker_threads self.thread_class = thread_class self.is_multiprocess = False if multiprocessing: if worker_processes == 'auto': cpus = multiprocessing.cpu_count() if cpus == 1: worker_processes = 0 else: worker_processes = cpus else: worker_processes = int(worker_processes) self.is_multiprocess = worker_processes > 0 self.worker_processes = worker_processes self.pipes = [] # used for sending management tasks # to subrpocesses self.task_dispatchers = [] # used for getting completed requests # from subprocesses only when sockets are # not pickleable self.sentinel = None self._socket = None request_queue = None done_queue = None if self.is_multiprocess: if not hasattr(socket, 'fromfd'): # create queues for communicating request_queue = get_shared_queue(2048) done_queue = get_shared_queue(2048) self.sentinel = (-1, b'EOF') else: request_queue = queue.Queue(worker_threads * 2) Dispatcher.__init__(self, request_queue, done_queue) # create worker tuple self.worker_pool = []
def start(self): # start runtime services BaseService.start(self) self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._socket.setblocking(0) try: self._socket.bind(self.addr) except socket.error as v: self._socket.close() raise v self._socket.listen(64) if self.request_queue is not None: # activate server socket self.set_socket(self._socket) # start workers self._start_workers() self.running = True
def __init__(self, name, worker_threads, thread_class, connection, request_queue=None, done_queue=None, sentinel=None, socket=None, local=None, master=None): BaseService.__init__(self, name) multiprocessing.Process.__init__(self, name=name) self.worker_threads = worker_threads self.thread_class = thread_class self.connection = connection self.request_queue = request_queue self.done_queue = done_queue self.sentinel = sentinel self.socket = socket self.is_alive = True self.master = master self.local = local
def shutdown(self): if self.running: self.running = False self._stop_workers() # shut down runtime services BaseService.shutdown(self)
def unlock_db(self): BaseService.unlock_db(self) if self.is_multiprocess: self.send("DB_UNLOCK")
def lock_db(self): BaseService.lock_db(self) if self.is_multiprocess: self.send("DB_LOCK")
def remove_runtime_service(self, component): if self.is_multiprocess and component == "db": self.send("DB_CLOSE") BaseService.remove_runtime_service(self, component)
def add_runtime_service(self, component, *args, **kwargs): inited = BaseService.add_runtime_service(self, component, *args, **kwargs) if self.is_multiprocess and component == "db": self.send("DB_OPEN") return inited
def start(self): BaseService.start(self) self.thread = Thread(name='%s thread' % self.name, target=self.thread_loop) self.running = True self.thread.start()
def run(self): # start runtime services BaseService.start(self) # set initial site master if self.master is not None: from porcupine.db import _db rep_mgr = _db.get_replication_manager() rep_mgr.master = self.master rep_mgr.local_site = self.local # start server if self.socket is not None: socket_map = {} # start server self.request_queue = queue.Queue(self.worker_threads * 2) self.done_queue = None server = Dispatcher(self.request_queue, None, socket_map) # activate server socket server.set_socket(self.socket, socket_map) # start asyncore loop asyn_thread = Thread(target=self._async_loop, args=(socket_map,), name="%s asyncore thread" % self.name) asyn_thread.start() else: # create queue for inactive RequestHandlerProxy objects # i.e. those served self.rhproxy_queue = queue.Queue(0) # patch shared queues self.request_queue = init_queue(*self.request_queue) self.done_queue = init_queue(*self.done_queue) thread_pool = [] for i in range(self.worker_threads): tname = "%s thread %d" % (self.name, i + 1) t = self.thread_class(target=self._thread_loop, name=tname) thread_pool.append(t) # start management thread mt = Thread(target=self._manage, name="%s management thread" % self.name) thread_pool.append(mt) # start threads [t.start() for t in thread_pool] try: while self.is_alive: time.sleep(8.0) except KeyboardInterrupt: pass except IOError: pass if self.socket is not None: for i in range(self.worker_threads): self.request_queue.put(self.sentinel) # join threads for t in thread_pool: t.join() if self.socket is not None: # join asyncore thread asyncore.close_all(socket_map) asyn_thread.join() # shutdown runtime services BaseService.shutdown(self)
def unlock_db(self): BaseService.unlock_db(self) if self.is_multiprocess: self.send('DB_UNLOCK')
def lock_db(self): BaseService.lock_db(self) if self.is_multiprocess: self.send('DB_LOCK')
def remove_runtime_service(self, component): if self.is_multiprocess and component == 'db': self.send('DB_CLOSE') BaseService.remove_runtime_service(self, component)
def add_runtime_service(self, component, *args, **kwargs): inited = BaseService.add_runtime_service(self, component, *args, **kwargs) if self.is_multiprocess and component == 'db': self.send('DB_OPEN') return inited
def __init__(self, name, interval): BaseService.__init__(self, name) self.interval = interval
def run(self): # start runtime services BaseService.start(self) # set initial site master if self.master is not None: from porcupine.db import _db rep_mgr = _db.get_replication_manager() rep_mgr.master = self.master rep_mgr.local_site = self.local # start server if self.socket is not None: socket_map = {} # start server self.request_queue = queue.Queue(self.worker_threads * 2) self.done_queue = None server = Dispatcher(self.request_queue, None, socket_map) # activate server socket server.set_socket(self.socket, socket_map) # start asyncore loop asyn_thread = Thread(target=self._async_loop, args=(socket_map, ), name='%s asyncore thread' % self.name) asyn_thread.start() else: # create queue for inactive RequestHandlerProxy objects # i.e. those served self.rhproxy_queue = queue.Queue(0) # patch shared queues self.request_queue = init_queue(*self.request_queue) self.done_queue = init_queue(*self.done_queue) thread_pool = [] for i in range(self.worker_threads): tname = '%s thread %d' % (self.name, i + 1) t = self.thread_class(target=self._thread_loop, name=tname) thread_pool.append(t) # start management thread mt = Thread(target=self._manage, name='%s management thread' % self.name) thread_pool.append(mt) # start threads [t.start() for t in thread_pool] try: while self.is_alive: time.sleep(8.0) except KeyboardInterrupt: pass except IOError: pass if self.socket is not None: for i in range(self.worker_threads): self.request_queue.put(self.sentinel) # join threads for t in thread_pool: t.join() if self.socket is not None: # join asyncore thread asyncore.close_all(socket_map) asyn_thread.join() # shutdown runtime services BaseService.shutdown(self)
def shutdown(self): self.running = False self.thread.join() BaseService.shutdown(self)