def test_iostat(self):
force_remove(config.iostat_stat_path)
with ututil.Timer() as t:
rst = fsutil.iostat('/dev/sda1')
dd(rst)
self.assertIsNotNone(rst)
self.assertEqual(set(rst_keys),
set(rst.keys()))
for k in rst_keys:
self.assertGreaterEqual(rst[k], 0)
self.assertGreaterEqual(t.spent(), 1.0)
# read again
with ututil.Timer() as t:
rst = fsutil.iostat('/dev/sda1')
dd(rst)
self.assertIsNotNone(rst)
self.assertEqual(set(rst_keys),
set(rst.keys()))
for k in rst_keys:
self.assertGreaterEqual(rst[k], 0)
self.assertAlmostEqual(0.0, t.spent(), delta=0.1)
def test_duration_is_negative(self):
force_remove(config.iostat_stat_path)
with ututil.Timer() as t:
rst = fsutil.iostat('/dev/sda1')
dd(rst)
self.assertIsNotNone(rst)
self.assertEqual(set(rst_keys),
set(rst.keys()))
self.assertGreaterEqual(t.spent(), 1.0)
d = shelve.open(config.iostat_stat_path)
x = d['sda1']
x['ts'] += 2
d['sda1'] = x
d.close()
with ututil.Timer() as t:
rst = fsutil.iostat('/dev/sda1')
dd(rst)
self.assertIsNotNone(rst)
self.assertEqual(set(rst_keys),
set(rst.keys()))
self.assertGreater(rst['write'], 0)
self.assertGreaterEqual(t.spent(), 1.0)
def test_daemon_after(self):
def _do():
pass
with ututil.Timer() as t:
th = tu.start_daemon(target=_do, after=None)
th.join()
self.assertAlmostEqual(0, t.spent(), delta=0.1)
with ututil.Timer() as t:
th = tu.start_daemon(target=_do, after=0.5)
th.join()
self.assertAlmostEqual(0.5, t.spent(), delta=0.1)
def test_wait_absent_change_node(self):
self.zk.create('a')
change_after = 0.2
for wait_time in (
0.5,
1,
):
dd('node present wait:', wait_time)
expected = max([0, wait_time])
def _change():
time.sleep(change_after)
self.zk.set('a', 'bbb')
th = threadutil.start_daemon(target=_change)
with ututil.Timer() as t:
self.assertRaises(zkutil.ZKWaitTimeout,
zkutil.wait_absent,
self.zk,
'a',
timeout=wait_time)
self.assertAlmostEqual(expected, t.spent(), delta=0.1)
th.join()
self.zk.delete('a')
def test_get_next_changed(self):
cases = (
0.4,
1,
)
def _set_a():
self.zk.set('a', 'changed')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_set_a, after=0.3)
with ututil.Timer() as t:
val, zstat = zkutil.get_next(self.zk,
'a',
timeout=timeout,
version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
self.assertEqual('changed', val)
self.assertEqual(1, zstat.version)
th.join()
self.zk.delete('a')
def test_ioutil_heavy_load(self):
sess = {'running': True}
def _write(i):
while sess['running']:
with open('/tmp/pykit-test-write-' + str(i), 'w') as f:
f.write('a' * 1024*1024*10)
th1 = threadutil.start_daemon(_write, args=(1, ))
th2 = threadutil.start_daemon(_write, args=(2, ))
time.sleep(5)
force_remove('/tmp/pykit-iostat')
with ututil.Timer() as t:
rst = fsutil.iostat('/dev/sda1')
dd(rst)
self.assertAlmostEqual(100, rst['ioutil'], delta=40)
self.assertGreaterEqual(t.spent(), 1.0)
sess['running'] = False
th1.join()
th2.join()
force_remove('/tmp/pykit-test-write-1')
force_remove('/tmp/pykit-test-write-2')
def test_get_next_changed_but_unsatisfied(self):
cases = (
0.4,
1,
)
def _set_a():
self.zk.set('a', 'changed')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_set_a, after=0.3)
with ututil.Timer() as t:
self.assertRaises(zkutil.ZKWaitTimeout,
zkutil.get_next,
self.zk,
'a',
timeout=timeout,
version=5)
self.assertAlmostEqual(timeout, t.spent(), delta=0.2)
th.join()
self.zk.delete('a')
def test_get_next_deleted(self):
cases = (
0.4,
1,
)
def _del_a():
self.zk.delete('a')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_del_a, after=0.3)
with ututil.Timer() as t:
self.assertRaises(NoNodeError,
zkutil.get_next,
self.zk,
'a',
timeout=timeout,
version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
th.join()
def test_bench(self):
pq = priorityqueue.PriorityQueue()
ntimes = 10240
nq = 1024
n_thread = 3
ths = []
def worker():
for _ in range(ntimes):
pq.get()
for i in range(1, nq + 1):
pq.add_producer(i, i, yield_forever())
with ututil.Timer() as t:
for i in range(n_thread):
th = threadutil.start_daemon_thread(worker)
ths.append(th)
for th in ths:
th.join()
us_per_call = t.spent() / ntimes / n_thread * 1000 * 1000
dd(us_per_call, 'us/call')
self.assertLess(us_per_call, 50)
def test_timeout(self):
l1 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
with l1:
with ututil.Timer() as t:
self.assertRaises(zkutil.LockTimeout, l2.acquire, timeout=0.2)
self.assertAlmostEqual(0.2, t.spent(), places=1)
with ututil.Timer() as t:
self.assertRaises(zkutil.LockTimeout, l2.acquire, timeout=-1)
self.assertAlmostEqual(0.0, t.spent(), delta=0.01)
try:
l2.acquire(timeout=-1)
except zkutil.LockTimeout:
self.fail('timeout<0 should could acquire')
def test_concurrent_3_record(self):
n_tx = 10
ks = ('foo', 'bar', 'duu')
def _tx(i):
while True:
try:
with ZKTransaction(zkhost) as t1:
for ii in range(len(ks)):
k = ks[(ii+i) % len(ks)]
foo = t1.lock_get(k)
foo.v = foo.v or 0
foo.v += 1
t1.set(foo)
t1.commit()
dd(str(t1) + ' committed')
return
except (Deadlock,
HigherTXApplied) as e:
dd(str(t1) + ': ' + repr(e))
continue
except TXTimeout as e:
dd(str(t1) + ': ' + repr(e))
raise
with ututil.Timer() as tt:
for th in [threadutil.start_daemon(_tx, args=(i, ))
for i in range(n_tx)]:
th.join()
dd('3 key 10 thread: ', tt.spent())
t = ZKTransaction(zkhost)
for k in ks:
rst, ver = t.zkstorage.record.get(k)
dd(rst)
self.assertEqual(n_tx, rst[-1][1])
rst, ver = t.zkstorage.txidset.get()
dd(rst)
self.assertEqual(n_tx, rst[COMMITTED].length())
# check aborted txidset
sm = rangeset.union(rst[COMMITTED], rst[PURGED])
self.assertEqual(rst[COMMITTED][-1][-1] - 1, sm.length())
def test_try_lock(self):
l1 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
with l1:
with ututil.Timer() as t:
locked, holder, ver = l2.try_acquire()
self.assertFalse(locked)
self.assertEqual(l1.identifier, holder)
self.assertGreaterEqual(ver, 0)
self.assertAlmostEqual(0.0, t.spent(), delta=0.05)
with ututil.Timer() as t:
locked, holder, ver = l2.try_acquire()
self.assertTrue(locked)
self.assertEqual(l2.identifier, holder)
self.assertEqual(ver, 0)
self.assertAlmostEqual(0.0, t.spent(), delta=0.05)
def test_get_next_no_version(self):
cases = (
-1,
0.0,
0.1,
1,
)
for timeout in cases:
self.zk.create('a', 'a-val')
with ututil.Timer() as t:
zkutil.get_next(self.zk, 'a', timeout=timeout, version=-1)
self.assertAlmostEqual(0, t.spent(), delta=0.2)
with ututil.Timer() as t:
zkutil.get_next(self.zk, 'a', timeout=timeout)
self.assertAlmostEqual(0, t.spent(), delta=0.2)
self.zk.delete('a')
def test_wait_absent(self):
for wait_time in (
-1,
0.0,
0.1,
1,
):
dd('no node wait:', wait_time)
with ututil.Timer() as t:
zkutil.wait_absent(self.zk, 'a', wait_time)
self.assertAlmostEqual(0, t.spent(), delta=0.2)
def test_get_next_conn_lost(self):
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=self.zk.stop, after=0.3)
with ututil.Timer() as t:
self.assertRaises(ConnectionClosedError,
zkutil.get_next,
self.zk,
'a',
timeout=1,
version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
th.join()
def test_wait_absent_connection_lost(self):
self.zk.create('a')
def _close():
time.sleep(.3)
self.zk.stop()
th = threadutil.start_daemon(target=_close)
with ututil.Timer() as t:
self.assertRaises(ConnectionClosedError, zkutil.wait_absent,
self.zk, 'a')
self.assertAlmostEqual(.3, t.spent(), delta=0.1)
th.join()
def test_wait_absent_no_timeout(self):
def _del():
time.sleep(1)
self.zk.delete('a')
for kwargs in (
{},
{
'timeout': None
},
):
self.zk.create('a')
th = threadutil.start_daemon(target=_del)
with ututil.Timer() as t:
zkutil.wait_absent(self.zk, 'a', **kwargs)
self.assertAlmostEqual(1, t.spent(), delta=0.1)
th.join()
def test_wait_absent_timeout(self):
self.zk.create('a')
for wait_time in (
-1,
0.0,
0.1,
1,
):
dd('node present wait:', wait_time)
expected = max([0, wait_time])
with ututil.Timer() as t:
self.assertRaises(zkutil.ZKWaitTimeout,
zkutil.wait_absent,
self.zk,
'a',
timeout=wait_time)
self.assertAlmostEqual(expected, t.spent(), delta=0.2)
self.zk.delete('a')
def test_wait_absent_delete_node(self):
delete_after = 0.2
for wait_time in (
0.5,
1,
):
dd('node present wait:', wait_time)
self.zk.create('a')
def _del():
time.sleep(delete_after)
self.zk.delete('a')
th = threadutil.start_daemon(target=_del)
with ututil.Timer() as t:
zkutil.wait_absent(self.zk, 'a', wait_time)
self.assertAlmostEqual(delete_after, t.spent(), delta=0.1)
th.join()
def test_config(self):
force_remove('/tmp/foo')
# write to default place and then ensure config changes stat path
rst = fsutil.iostat('/dev/sda1')
# change stat path
old = config.iostat_stat_path
config.iostat_stat_path = '/tmp/foo'
with ututil.Timer() as t:
rst = fsutil.iostat('/dev/sda1')
dd(rst)
self.assertGreaterEqual(t.spent(), 1.0)
# should be able to read something.
fsutil.read_file(config.iostat_stat_path)
force_remove(config.iostat_stat_path)
config.iostat_stat_path = old
def test_get_next_timeout(self):
cases = (
-1,
0.0,
0.2,
1,
)
for timeout in cases:
expected = max([timeout, 0])
self.zk.create('a', 'a-val')
with ututil.Timer() as t:
self.assertRaises(zkutil.ZKWaitTimeout,
zkutil.get_next,
self.zk,
'a',
timeout=timeout,
version=0)
self.assertAlmostEqual(expected, t.spent(), delta=0.2)
self.zk.delete('a')
