def run(self, env, json_writer):
'''
Schedule all tests in profile for execution.
See ``Test.schedule`` and ``Test.run``.
'''
self.prepare_test_list(env)
# If using concurrency, add all the concurrent tests to the pool and
# execute that pool
if env.concurrent:
pool = ThreadPool(multiprocessing.cpu_count())
for (path, test) in self.test_list.items():
if test.runConcurrent:
pool.add(test.execute, (env, path, json_writer))
pool.join()
# Run any remaining tests serially from a single thread pool after the
# concurrent tests have finished
pool = ThreadPool(1)
for (path, test) in self.test_list.items():
if not env.concurrent or not test.runConcurrent:
pool.add(test.execute, (env, path, json_writer))
pool.join()
def run(self, env, json_writer):
'''
Schedule all tests in profile for execution.
See ``Test.schedule`` and ``Test.run``.
'''
self.prepare_test_list(env)
# If using concurrency, add all the concurrent tests to the pool and
# execute that pool
if env.concurrent:
pool = ThreadPool(multiprocessing.cpu_count())
for (path, test) in self.test_list.items():
if test.runConcurrent:
pool.add(test.execute, (env, path, json_writer))
pool.join()
# Run any remaining tests serially from a single thread pool after the
# concurrent tests have finished
pool = ThreadPool(1)
for (path, test) in self.test_list.items():
if not env.concurrent or not test.runConcurrent:
pool.add(test.execute, (env, path, json_writer))
pool.join()
def thread_run():
# do_trans_all_logger()
def run(obj, client, addr):
return handle(client, addr)
t = ThreadPool(max_thread)
t.start()
while True:
try:
client, addr = sock.accept()
t.add(Task(run, client=client, addr=addr))
print t.queue.qsize()
except KeyboardInterrupt:
os.kill(os.getpid(), 9)
#except Queue.Full:
# client.close()
except:
client.close()
log.debug(traceback.format_exc())
class ThriftThreadServer:
def __init__(self, module, handler_class, addr, max_process=1, max_conn=50, framed=True):
module.handler = handler_class()
global service
service = module
self.framed = framed
self.workers = []
self.running = True
def signal_master_handler(signum, frame):
log.warn("signal %d catched in master %d, wait for kill all worker", signum, os.getpid())
self.running = False
for p in self.workers:
p.terminate()
def signal_worker_handler(signum, frame):
log.warn("worker %d will exit after all request handled", os.getpid())
#self.server.close()
self.running = False
self.sock.close()
def server_init():
signal.signal(signal.SIGTERM, signal_worker_handler)
log.warn('server started addr=%s:%d pid=%d', addr[0], addr[1], os.getpid())
if hasattr(service.handler, '_initial'):
service.handler._initial()
#self.server.serve_forever()
def _start_process(index):
server_name = 'proc-%02d' % index
p = multiprocessing.Process(target=server_start, name=server_name)
p.start()
return p
def signal_child_handler(signum, frame):
time.sleep(1)
if self.running:
log.warn("master recv worker exit, fork one")
try:
pinfo = os.waitpid(-1, 0)
pid = pinfo[0]
index = -1
for i in range(0, len(self.workers)):
p = self.workers[i]
if p.pid == pid:
index = i
break
if index >= 0:
self.workers[index] = _start_process(index)
except OSError:
log.info('waitpid error:')
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(addr)
self.sock.listen(1024)
server_init()
self.tp = ThreadPool(max_conn, max_conn*5)
self.tp.start()
def forever(self):
class MyTask (Task):
def run(self):
return self._func(*self._args, **self._kwargs)
while self.running:
try:
client, addr = self.sock.accept()
self.tp.add(MyTask(handle, client=client, addr=addr, framed=self.framed))
except KeyboardInterrupt:
break
#os.kill(os.getpid(), 9)
except Exception, e:
log.warn(traceback.format_exc())
log.warn('master exception: %s', str(e))
self.sock.close()
self.sock = None
self.tp.stop_wait()
log.warn('master exit')
