def run():
try:
register.register((100003, 3, IPPROTO_TCP, 2049))
register.register((100005, 3, IPPROTO_TCP, 5555))
th_mountd = threadutil.start_daemon_thread(mountd)
th_nfsd = threadutil.start_daemon_thread(nfsd)
th_mountd.join()
th_nfsd.join()
except KeyboardInterrupt:
sys.exit(0)
def test_brecv_message(self):
c = redisutil.RedisChannel((self.ip, redis_port), '/foo', 'client')
s = redisutil.RedisChannel((self.ip, redis_port), '/foo', 'server')
c.send_msg('aa')
self.assertEqual('aa', s.brecv_msg(timeout=1))
self.assertEqual(None, s.brecv_msg(timeout=1))
def _send_msg():
time.sleep(0.5)
c.send_msg('bar')
threadutil.start_daemon_thread(target=_send_msg)
self.assertEqual('bar', s.brecv_msg(timeout=1))
def test_wait_for_data_timeout(self):
expected = [
'a' * 32,
'b' * 32,
]
rst = []
append_lines(self.fn, expected)
dd('appended 1')
def _append():
time.sleep(0.3)
append_lines(self.fn, expected)
dd('appended')
th = threadutil.start_daemon_thread(_append)
try:
for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=0.1):
rst.append(l)
self.failed('expect NoData to raise')
except fsutil.NoData:
pass
th.join()
self.assertEqual(expected, rst)
def test_concurrent(self):
pq = priorityqueue.PriorityQueue()
ntimes = 10240
nq = 3
n_thread = 3
ths = []
def worker():
for _ in range(ntimes):
pq.get()
for i in range(1, nq + 1):
pq.add_producer(i, i, yield_forever())
for i in range(n_thread):
th = threadutil.start_daemon_thread(worker)
ths.append(th)
for th in ths:
th.join()
consumed = []
got = 0
for i in range(1, nq + 1):
q = pq.producer_by_id[i]
consumed.append(q.consumed)
dd('get:', q.stat)
got += q.stat['get']
self.assertEqual(ntimes * n_thread, got)
dd('consumed:', consumed)
self.assertAlmostEqual(consumed[0], consumed[1])
self.assertAlmostEqual(consumed[1], consumed[2])
def test_bench(self):
pq = priorityqueue.PriorityQueue()
ntimes = 10240
nq = 1024
n_thread = 3
ths = []
def worker():
for _ in range(ntimes):
pq.get()
for i in range(1, nq + 1):
pq.add_producer(i, i, yield_forever())
with ututil.Timer() as t:
for i in range(n_thread):
th = threadutil.start_daemon_thread(worker)
ths.append(th)
for th in ths:
th.join()
us_per_call = t.spent() / ntimes / n_thread * 1000 * 1000
dd(us_per_call, 'us/call')
self.assertLess(us_per_call, 50)
def test_set_thread_num_keep_order(self):
def _pass(args):
return args
rst = []
jm = jobq.JobManager([_pass, rst.append], keep_order=True)
setter = {'running': True}
def _change_thread_nr():
while setter['running']:
jm.set_thread_num(_pass, random.randint(1, 4))
time.sleep(0.5)
ths = []
for ii in range(3):
th = threadutil.start_daemon_thread(_change_thread_nr)
ths.append(th)
n = 10240
for i in range(n):
jm.put(i)
jm.join()
rst.sort()
for i in range(n):
self.assertEqual(i, rst[i])
setter['running'] = False
for th in ths:
th.join()
def fire(self, curr_time, job_name, job_conf, job_status):
thread_n = len(job_status['active_threads'])
if thread_n >= job_conf['concurrence_n']:
log_msg = 'at time: %s, already have %d threads for job: %s' % (
curr_time, thread_n, job_name)
self.append_log(job_status, job_conf, log_msg)
logger.error('too many threads for job: %s' % job_name)
return
logger.info('at time: %s, start to run job: %s' %
(curr_time, job_name))
threadutil.start_daemon_thread(
self.run_job, args=(curr_time, job_name, job_conf, job_status))
job_status['message'] = ''
def run(**argkv):
context = {
'get_cgroup_pid_file': argkv['get_cgroup_pid_file'],
'cgroup_dir': argkv.get('cgroup_dir', '/sys/fs/cgroup'),
'communicate_ip': argkv.get('communicate_ip', '0.0.0.0'),
'communicate_port': argkv.get('communicate_port', 43409),
'tasks_update_interval': argkv.get('tasks_update_interval', 30),
'redis_ip': argkv['redis_ip'],
'redis_port': argkv['redis_port'],
'redis_prefix': argkv.get('redis_prefix', 'cgroup_arch'),
'redis_client': None,
'redis_expire_time': argkv.get('redis_expire_time', 60 * 5),
'get_zk_host': argkv['get_zk_host'],
'zk_prefix': argkv['zk_prefix'],
'zk_auth_data': argkv['zk_auth_data'],
'zk_client': None,
'protected_cgroup': argkv.get('protected_cgroup'),
'arch_conf': None,
}
init_redis_client(context)
init_arch_conf(context)
global_value['context'] = context
cgroup_manager.build_all_subsystem_cgroup_arch(context)
cgroup_manager.set_cgroup(context)
cgroup_manager.reset_statistics(context)
threadutil.start_daemon_thread(account.run, args=(context,))
threadutil.start_daemon_thread(cgroup_manager.loop_set_cgroup,
args=(context,))
communicate.run(context, ip=context['communicate_ip'],
port=context['communicate_port'])
def run(**argkv):
context = {
# websocket listen ip and port
'ip': argkv.get('ip', '127.0.0.1'),
'port': argkv.get('port', 32345),
# every message received will be put into messgee queue,
# and processed by message processor
'message_queue': Queue.Queue(1024 * 10),
# info in destribution will be used by function assign()
'distribution': {},
# for monitoring
'connections': {},
'reported': {},
'consumption_sum': {},
'rejection_sum': {},
'quota': {},
# specify how many slot to save in a dict contain info by slot
# normally, only recent slots should be kept
'nr_slot': argkv.get('nr_slot', 60),
# a global lock class which can be used in 'with' statement.
# it is used to make sure only one central node acutally working
# at any time.
'Lock': argkv['Lock'],
# who got lock, who running
'running': False,
# callback function to get all limits setted for a service
'list_limits': argkv['list_limits'],
}
threadutil.start_daemon_thread(assigner.run, args=(context, ))
threadutil.start_daemon_thread(message_processor.run, args=(context, ))
communicate.run(context)
def start_assign(context, assign_threads, slot_number):
for service_name, service_model in service.services.iteritems():
if service_name in assign_threads:
continue
th = threadutil.start_daemon_thread(service_model['module'].assign,
args=(
context,
slot_number,
))
assign_threads[service_name] = th
def run(**kwargs):
context = {
'node_id': kwargs['node_id'],
'node_ip': kwargs['node_ip'],
'send_log': kwargs['send_log'],
'conf': kwargs['conf'],
'cache_lock': threading.RLock(),
'cache': {},
'stat': {},
'queue': Queue.Queue(1024 * 10),
}
# strptime not thread safe, need to call it manually before
# initiating any thread
datetime.strptime("2011-04-05", "%Y-%m-%d")
for log_name in context['conf'].keys():
context['cache'][log_name] = {}
context['stat'][log_name] = {}
threadutil.start_daemon_thread(scanner.scan, args=(context, log_name))
threadutil.start_daemon_thread(cache_flusher.run, args=(context, ))
threadutil.start_daemon_thread(sender.run, args=(context, ))
while True:
# actually it is not an error log, but normally we only report
# error log, and we want to report this log even it is not
# an error log.
logger.error('stat: %s' % context['stat'])
time.sleep(100)
def run(**argkv):
context = {
'get_cgroup_pid_file': argkv['get_cgroup_pid_file'],
'cgroup_dir': argkv.get('cgroup_dir', '/sys/fs/cgroup'),
'communicate_ip': argkv.get('communicate_ip', '0.0.0.0'),
'communicate_port': argkv.get('communicate_port', 43409),
'tasks_update_interval': argkv.get('tasks_update_interval', 30),
'redis_ip': argkv['redis_ip'],
'redis_port': argkv['redis_port'],
'redis_prefix': argkv.get('redis_prefix', 'cgroup_arch'),
'redis_client': None,
'redis_expire_time': argkv.get('redis_expire_time', 60 * 5),
'get_zk_host': argkv['get_zk_host'],
'zk_prefix': argkv['zk_prefix'],
'zk_auth_data': argkv['zk_auth_data'],
'zk_client': None,
'protected_cgroup': argkv.get('protected_cgroup'),
'arch_conf': None,
}
init_redis_client(context)
init_arch_conf(context)
global_value['context'] = context
cgroup_manager.build_all_subsystem_cgroup_arch(context)
cgroup_manager.set_cgroup(context)
cgroup_manager.reset_statistics(context)
threadutil.start_daemon_thread(account.run, args=(context, ))
threadutil.start_daemon_thread(cgroup_manager.loop_set_cgroup,
args=(context, ))
communicate.run(context,
ip=context['communicate_ip'],
port=context['communicate_port'])
def setUp(self):
self.addr = ('127.0.0.1', 22038)
self.proxy_addr = ('127.0.0.1', 22039)
self.response['http-status'] = 200
def _start_http_svr():
self.http_server = HTTPServer(self.addr, HttpHandle)
self.http_server.serve_forever()
def _start_proxy_http_svr():
self.proxy_http_server = HTTPServer(self.proxy_addr, HttpHandle)
self.proxy_http_server.serve_forever()
threadutil.start_daemon_thread(_start_http_svr)
threadutil.start_daemon_thread(_start_proxy_http_svr)
time.sleep(0.1)
self.cli = redisutil.RedisProxyClient([self.addr])
self.n = self.cli.n
self.w = self.cli.w
self.r = self.cli.r
def test_file_end_handler(self):
expected = [
'a' * 32,
'b' * 32,
]
append_lines(self.fn, expected)
rst = []
def _end():
rst.append('end')
# file_end_handler in cat()
c = fsutil.Cat(self.fn,
strip=True,
handler=rst.append,
file_end_handler=_end)
c.cat(timeout=0)
self.assertEqual(expected + ['end'], rst)
# file_end_handler in iterate()
rst = []
force_remove(self.fn)
append_lines(self.fn, expected)
for line in fsutil.Cat(self.fn, strip=True,
file_end_handler=_end).iterate(timeout=0):
rst.append(line)
self.assertEqual(expected + ['end'], rst)
# file_end_handler multi times
rst = []
force_remove(self.fn)
append_lines(self.fn, expected)
def _append():
time.sleep(0.1)
append_lines(self.fn, expected)
th = threadutil.start_daemon_thread(_append)
try:
for line in fsutil.Cat(self.fn, strip=True,
file_end_handler=_end).iterate(timeout=0.2):
rst.append(line)
except fsutil.NoData:
pass
th.join()
self.assertEqual((expected + ['end']) * 2, rst)
def test_concurrent(self):
self.running = True
self.total = 0
n_repeat = 40
n_thread = 5
ths = []
for ii in range(n_thread):
t = threadutil.start_daemon_thread(self._loop_acquire, args=(n_repeat, ii,))
ths.append(t)
for th in ths:
th.join()
self.running = False
self.assertEqual(n_repeat * n_thread, self.total)
def start_thread(exec_func, *args):
# `thread_index` identifying a thread in worker_group.threads.
# It is used to decide which thread to remove.
return threadutil.start_daemon_thread(exec_func, args=args)
def test_write_file_atomically(self):
fn = '/tmp/pykit-ut-fsutil-write-atomic'
dd('atomically write file')
cont_thread1 = 'cont_thread1'
cont_thread2 = 'cont_thread2'
os_fsync = os.fsync
def _wait_fsync(fildes):
time.sleep(3)
os_fsync(fildes)
os.fsync = _wait_fsync
assert_ok = {'ok': True}
def _write_wait(cont_write, cont_read, start_after, atomic):
time.sleep(start_after)
fsutil.write_file(fn, cont_write, atomic=atomic)
if cont_read != fsutil.read_file(fn):
assert_ok['ok'] = False
force_remove(fn)
# atomic=False
# time file thread1 thread2
# 0 cont_1 w_cont_1 sleep()
# 1.5 cont_2 sleep() w_cont_2
# 3 cont_2 return sleep()
# 4.5 cont_2 None return
ths = []
th = threadutil.start_daemon_thread(_write_wait,
args=(cont_thread1, cont_thread2,
0, False))
ths.append(th)
th = threadutil.start_daemon_thread(_write_wait,
args=(cont_thread2, cont_thread2,
1.5, False))
ths.append(th)
for th in ths:
th.join()
self.assertTrue(assert_ok['ok'])
force_remove(fn)
# atomic=True
# time file thread1 thread2
# 0 None w_cont_1 sleep()
# 1.5 None sleep() w_cont_2
# 3 cont_1 return sleep()
# 4.5 cont_2 None return
ths = []
th = threadutil.start_daemon_thread(_write_wait,
args=(cont_thread1, cont_thread1,
0, True))
ths.append(th)
th = threadutil.start_daemon_thread(_write_wait,
args=(cont_thread2, cont_thread2,
1.5, True))
ths.append(th)
for th in ths:
th.join()
self.assertTrue(assert_ok['ok'])
os.fsync = os_fsync
force_remove(fn)
def test_basic(self):
logger = logutil.make_logger(base_dir=this_base, log_name='test_log')
def log():
start_time = time.time()
while True:
logger.info('info')
logger.warn('warn')
logger.error('error')
if time.time() > start_time + 2.5:
break
time.sleep(0.01)
log_th = threadutil.start_daemon_thread(log)
log_entries = []
def get_level(log_str):
for k in ('INFO', 'WARNING', 'ERROR'):
if k in log_str:
return k.lower()
else:
return 'unknown'
def send_log(log_entry):
log_entries.append(log_entry)
kwargs = {
'node_id': '123abc',
'node_ip': '1.2.3.4',
'send_log': send_log,
'conf': {
'my_test_log': {
'file_path': os.path.join(this_base, 'test_log.out'),
'level': ['error'],
'get_level': get_level,
'is_first_line': is_first_line,
'parse': parse,
},
},
}
threadutil.start_daemon_thread(collector.run, kwargs=kwargs)
log_th.join()
time.sleep(2)
self.assertEqual(3, len(log_entries))
dd(log_entries)
dd(log_entries[0]['count'])
dd(log_entries[1]['count'])
dd(log_entries[2]['count'])
self.assertAlmostEqual(100, log_entries[1]['count'], delta=30)
self.assertEqual('error', log_entries[0]['level'])
self.assertEqual('my_test_log', log_entries[0]['log_name'])
self.assertEqual('test_log.out', log_entries[0]['log_file'])
def start_thread(exec_func, *args):
# `thread_index` identifying a thread in worker_group.threads.
# It is used to decide which thread to remove.
return threadutil.start_daemon_thread(exec_func, args=args)
