def __init__(self, pool_size=4, task_size=100):
super(EventBus, self).__init__()
self.pool = ThreadPool(pool_size)
self.async_events = TaskPool(task_size)
self.event_handlers = dict()
self.con = Condition()
self.init()
def run_tests(tests, summary):
jobs = OPTIONS.workercount
# python 3.3 fixed a bug with concurrently writing .pyc files.
# https://bugs.python.org/issue13146
embedded_version = (subprocess.check_output([
OPTIONS.gdb_executable,
"--batch",
"--ex",
"python import sys; print(sys.hexversion)",
]).decode("ascii").strip())
if hex(int(embedded_version)) < "0x3030000":
jobs = 1
pool = TaskPool(tests, job_limit=jobs, timeout=OPTIONS.timeout)
pool.run_all()
def __worker_prepare(self, taskpool=None):
Worker.setWorkerConfig(
rootlogger=LOGGER,
basedomain=self.__config["--basedomain"]
if self.__config["--basedomain"] else None,
errorpage=self.__config["--errorpage"]
if self.__config["--errorpage"] else None,
taskpool=TaskPool(distinct_filter_dump="filter_dump_{now}".format(
now=NOW)) if not taskpool else taskpool)
if not self.resume:
Worker.Worker_taskpool.put(self.__config["<src>"])
if self.__config["--debug"]:
map(Worker.add_debuglog_handler,
(getHandler(logging.FileHandler, filename=dbg_log_file)
for dbg_log_file in self.__config["--dbglog"]))
Worker.add_debuglog_handler(getHandler(logging.StreamHandler))
if self.__config["--syslog"]:
map(Worker.add_syslog_handler,
(getHandler(hdlr_cls=logging.FileHandler,
filename=sys_log_file)
for sys_log_file in self.__config["--syslog"]))
elif not self.__config["--syslog"] and not self.__config["--debug"]:
Worker.add_syslog_handler(
getHandler(hdlr_cls=logging.StreamHandler))
if self.__config["--errlog"]:
map(Worker.add_errorlog_handler,
(getHandler(hdlr_cls=logging.FileHandler,
filename=err_log_file)
for err_log_file in self.__config["--errlog"]))
elif not self.__config["--errlog"] and not self.__config["--debug"]:
Worker.add_errorlog_handler(
getHandler(hdlr_cls=logging.StreamHandler))
def run_tests(tests, summary):
pool = TaskPool(tests,
job_limit=OPTIONS.workercount,
timeout=OPTIONS.timeout)
pool.run_all()
import threading
import os
import sys
import json
from BaseHTTPServer import HTTPServer
from taskpool import TaskPool
from HTTPlistener import httpServerFactory
f = open('taskpool_config', 'r')
master_config = json.load(f)
f.close
tp = TaskPool(master_config)
taskpool_thread = threading.Thread(target=tp.loop, args=()).start()
ip = master_config['master address']
port = master_config['master port']
HTTPListener = httpServerFactory({'taskpool': tp})
serv = HTTPServer((ip, port), HTTPListener)
print 'Starting master http server at port %d ....'%master_config['master port']
try:
serv.serve_forever()
except KeyboardInterrupt:
tp.close_connection()
print 'KeyboardInterrupt 8P'
os._exit(0)
return 'content for key %s: %s' % (inner_key, random.randint(
1, 100000))
content = read_cached(inner_open_test_random, key)
return content
if __name__ == '__main__':
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s:%(name)s:%(levelname)s:%(message)s')
logging.getLogger('requests').setLevel(logging.WARNING)
file_handler = logging.FileHandler('download-vessels-details.log',
mode='w')
formatter = logging.Formatter(
'%(asctime)s:%(name)s:%(levelname)s:%(message)s')
file_handler.setFormatter(formatter)
logging.getLogger().addHandler(file_handler)
set_cache_path('output/tests', max_node_files=400, rebalancing_limit=1000)
delete_cache()
tasks = TaskPool(30)
for count in range(10000):
tasks.add_task(open_test_random, count)
results = tasks.execute()
logging.info('results: %s', results)
delete_cache()
