def run(self, env, json_writer):
'''
Schedule all tests in profile for execution.
See ``Test.schedule`` and ``Test.run``.
'''
self.prepare_test_list(env)
# If using concurrency, add all the concurrent tests to the pool and
# execute that pool
if env.concurrent:
pool = ThreadPool(multiprocessing.cpu_count())
for (path, test) in self.test_list.items():
if test.runConcurrent:
pool.add(test.execute, (env, path, json_writer))
pool.join()
# Run any remaining tests serially from a single thread pool after the
# concurrent tests have finished
pool = ThreadPool(1)
for (path, test) in self.test_list.items():
if not env.concurrent or not test.runConcurrent:
pool.add(test.execute, (env, path, json_writer))
pool.join()
def run(self, env, json_writer):
'''
Schedule all tests in profile for execution.
See ``Test.schedule`` and ``Test.run``.
'''
self.prepare_test_list(env)
# If using concurrency, add all the concurrent tests to the pool and
# execute that pool
if env.concurrent:
pool = ThreadPool(multiprocessing.cpu_count())
for (path, test) in self.test_list.items():
if test.runConcurrent:
pool.add(test.execute, (env, path, json_writer))
pool.join()
# Run any remaining tests serially from a single thread pool after the
# concurrent tests have finished
pool = ThreadPool(1)
for (path, test) in self.test_list.items():
if not env.concurrent or not test.runConcurrent:
pool.add(test.execute, (env, path, json_writer))
pool.join()
def main(hostnames=[], repeat=1):
pool = ThreadPool(TraceRoute.trace)
for i in range(0, 1):
for x in hostnames:
pool.submit(x)
pool.join()
g = Graph(pool)
g.draw()
plt.show()
def processing(tables):
import opts
options, _ = opts.parse_options()
master_connect_setting, slaver_connect_setting = opts.parse_connection(options)
master = SqlPool(**master_connect_setting)
slaver = SqlPool(**slaver_connect_setting)
if not tables and options.tables:
tables = options.tables
if not tables:
tables = show_tables(slaver)
pool = ThreadPool(options.thread)
for tb in tables:
ignored = False
for i in options.tables_ignored:
if tb.startswith(i):
ignored = True
break
if ignored: continue
if tb not in ['qing_filerecycle', 'qing_file_storage'] and \
options.checksum and checksum_table(master, slaver, tb) == True:
logging.warn("Table[%s] on slave was the smae as master . OK", tb)
continue
checker = TableChecker(master, slaver, tb,
setp = 1000,
used_select_all = bool(tb in options.tables_select_all),
fix_diff=options.fix_diff)
pool.put(checker.run)
pool.join()
def worker_routine(context, idx):
""" Worker routine """
global worker_exe_path
# Socket to talk to dispatcher
socket0 = context.socket(zmq.REP)
socket0.connect(url_worker)
worker_id = 'Worker-%s' % idx
socket0.setsockopt(zmq.IDENTITY, worker_id)
socket_exit = context.socket(zmq.SUB)
socket_exit.connect(url_worker_cmd)
socket_exit.setsockopt(zmq.SUBSCRIBE, '')
poller = zmq.Poller()
poller.register(socket0, zmq.POLLIN)
poller.register(socket_exit, zmq.POLLIN)
from threadpool import ThreadPool, WorkRequest
thread_pool = ThreadPool(thread_num_of_worker)
def thread_work(client_exe_path, msg):
# create a process to execute the client_exe
# TODO: add execute timeout
_cmd = '%s %s' % (client_exe_path, msg)
cmd = WrapCommand(_cmd)
cmd.start()
logger.debug('Start command start %s' % _cmd)
cmd.join()
logger.debug('Start command end %s' % _cmd)
return cmd
def thread_work_cb(request, ret):
if ret.returncode == 0:
# run cmd process in thread success
logger.debug("execute %s success %s" %
(request.args[0], ret.returncode))
else:
logger.error(
"execute %s failed %s \n%s\n" %
(request.args[0], ret.returncode, ' '.join(ret.results)))
def handle_exception(request, exc_info):
if not isinstance(exc_info, tuple):
# Something is seriously wrong...
logger.debug(request)
logger.debug(exc_info)
raise SystemExit
logger.debug("**** Exception occured in request #%s: %s" % \
(request.requestID, exc_info))
def register_task(client_exe_path, msg):
request = WorkRequest(thread_work, (client_exe_path, msg), {},
callback=thread_work_cb,
exc_callback=handle_exception)
thread_pool.putRequest(request)
def get_client_exe_paths(client_name):
client_exe_paths = []
for client_id in CONF.clients.clients_list:
client_conf = getattr(CONF, client_id)
client_n = client_conf.name
if client_n == client_name:
client_exe_paths.append(client_conf.exe_path)
return client_exe_paths
while True:
events = dict(poller.poll(2000))
if events.get(socket0) == zmq.POLLIN:
# Deal with message
_msg = socket0.recv()
print("Received request: [%s]\n" % (_msg))
client_name = _msg.split()[0]
msg = _msg[len(client_name):].strip()
exe_paths = get_client_exe_paths(client_name)
logger.debug("Client name \"%s\" message: %s" % (client_name, msg))
logger.debug("Client exe path %s" % exe_paths)
if exe_paths is not None:
for exe_path in exe_paths:
register_task(exe_path, msg)
#send reply back to client
socket0.send("OK")
else:
logger.debug(
"Can not execute client, because there have none clients in config file(%s), please check"
% str(CONF.clients.clients_list))
socket0.send('FAILED')
if events.get(socket_exit) == zmq.POLLIN:
cmd = socket_exit.recv()
logger.debug('%s CMD %s' % (worker_id, cmd))
sys.stdout.flush()
if cmd == 'EXIT':
break
elif cmd == 'CONFIG_CHANGED':
reload_config()
else:
pass
thread_pool.close()
thread_pool.join()
socket0.close()
socket_exit.close()
def worker_routine(context, idx):
""" Worker routine """
global worker_exe_path
# Socket to talk to dispatcher
socket0 = context.socket(zmq.REP)
socket0.connect(url_worker)
worker_id = 'Worker-%s' % idx
socket0.setsockopt(zmq.IDENTITY, worker_id)
socket_exit = context.socket(zmq.SUB)
socket_exit.connect(url_worker_cmd)
socket_exit.setsockopt(zmq.SUBSCRIBE, '')
poller = zmq.Poller()
poller.register(socket0, zmq.POLLIN)
poller.register(socket_exit, zmq.POLLIN)
from threadpool import ThreadPool, WorkRequest
thread_pool = ThreadPool(thread_num_of_worker)
def thread_work(client_exe_path, msg):
# create a process to execute the client_exe
# TODO: add execute timeout
_cmd = '%s %s' % (client_exe_path, msg)
cmd = WrapCommand(_cmd)
cmd.start()
logger.debug('Start command start %s' % _cmd)
cmd.join()
logger.debug('Start command end %s' % _cmd)
return cmd
def thread_work_cb(request, ret):
if ret.returncode == 0:
# run cmd process in thread success
logger.debug("execute %s success %s" % (request.args[0], ret.returncode))
else:
logger.error("execute %s failed %s \n%s\n" % (request.args[0], ret.returncode, ' '.join(ret.results)))
def handle_exception(request, exc_info):
if not isinstance(exc_info, tuple):
# Something is seriously wrong...
logger.debug(request)
logger.debug(exc_info)
raise SystemExit
logger.debug("**** Exception occured in request #%s: %s" % \
(request.requestID, exc_info))
def register_task(client_exe_path, msg):
request = WorkRequest(thread_work, (client_exe_path, msg), {}, callback=thread_work_cb,
exc_callback=handle_exception)
thread_pool.putRequest(request)
def get_client_exe_paths(client_name):
client_exe_paths = []
for client_id in CONF.clients.clients_list:
client_conf = getattr(CONF, client_id)
client_n = client_conf.name
if client_n == client_name:
client_exe_paths.append(client_conf.exe_path)
return client_exe_paths
while True:
events = dict(poller.poll(2000))
if events.get(socket0) == zmq.POLLIN:
# Deal with message
_msg = socket0.recv()
print("Received request: [%s]\n" % (_msg))
client_name = _msg.split()[0]
msg = _msg[len(client_name):].strip()
exe_paths = get_client_exe_paths(client_name)
logger.debug("Client name \"%s\" message: %s" %(client_name, msg))
logger.debug("Client exe path %s" % exe_paths)
if exe_paths is not None:
for exe_path in exe_paths:
register_task(exe_path, msg)
#send reply back to client
socket0.send("OK")
else:
logger.debug("Can not execute client, because there have none clients in config file(%s), please check" % str(CONF.clients.clients_list))
socket0.send('FAILED')
if events.get(socket_exit) == zmq.POLLIN:
cmd = socket_exit.recv()
logger.debug('%s CMD %s' % (worker_id, cmd))
sys.stdout.flush()
if cmd == 'EXIT':
break
elif cmd == 'CONFIG_CHANGED':
reload_config()
else:
pass
thread_pool.close()
thread_pool.join()
socket0.close()
socket_exit.close()
