def test_basic(self):
log_entries = []
def send_log(log_entry):
log_entries.append(log_entry)
kwargs = {
'node_id': '123abc',
'node_ip': '1.2.3.4',
'send_log': send_log,
'conf': {
'my_test_log': {
'file_path': os.path.join(this_base, 'test_log.out'),
'level': ['error'],
'get_level': self.get_level,
'is_first_line': is_first_line,
'parse': parse,
},
},
}
threadutil.start_daemon(self.log)
threadutil.start_daemon(collector.run, kwargs=kwargs)
time.sleep(8)
dd(log_entries)
log_cnt = 0
for le in log_entries:
log_cnt += le["count"]
dd(log_cnt)
self.assertEqual(100, log_cnt)
self.assertEqual('error', log_entries[0]['level'])
self.assertEqual('my_test_log', log_entries[0]['log_name'])
self.assertEqual('test_log.out', log_entries[0]['log_file'])
def test_no_merge(self):
log_entries = []
def send_log(log_entry):
print log_entry
log_entries.append(log_entry)
kwargs = {
'node_id': '123abc',
'node_ip': '1.2.3.4',
'send_log': send_log,
'conf': {
'my_test_log': {
'file_path': os.path.join(this_base, 'test_log.out'),
'level': ['error'],
'get_level': self.get_level,
'is_first_line': is_first_line,
'parse': parse,
'merge': False,
},
},
}
threadutil.start_daemon(self.log)
threadutil.start_daemon(collector.run, kwargs=kwargs)
time.sleep(8)
self.assertEqual(100, len(log_entries))
self.assertEqual('error', log_entries[0]['level'])
def test_deadlock(self):
# deadlock is not recoverable
t = ZKTransaction(zkhost)
# t.txid is lower
t.begin()
t.lock_get('foo')
def _commit():
t.commit()
txid = None
try:
with ZKTransaction(zkhost, timeout=0.5) as t1:
txid = t1.txid
# lock another key first to produce deadlock
t1.lock_get('woo')
t1.set_state('bar')
threadutil.start_daemon(_commit, after=0.2)
t1.lock_get('foo') # should deadlock waiting for higher txid
except Deadlock:
pass
t = ZKTransaction(zkhost)
self.assertIsNone(t.zkstorage.state.get(txid)[0])
def test_ioutil_heavy_load(self):
sess = {'running': True}
def _write(i):
while sess['running']:
with open('/tmp/pykit-test-write-' + str(i), 'w') as f:
f.write('a' * 1024*1024*10)
th1 = threadutil.start_daemon(_write, args=(1, ))
th2 = threadutil.start_daemon(_write, args=(2, ))
time.sleep(5)
force_remove('/tmp/pykit-iostat')
with ututil.Timer() as t:
rst = fsutil.iostat('/dev/sda1')
dd(rst)
self.assertAlmostEqual(100, rst['ioutil'], delta=40)
self.assertGreaterEqual(t.spent(), 1.0)
sess['running'] = False
th1.join()
th2.join()
force_remove('/tmp/pykit-test-write-1')
force_remove('/tmp/pykit-test-write-2')
def test_deadlock(self):
# deadlock is not recoverable
t = ZKTransaction(zkhost)
# t.txid is lower
t.begin()
t.lock_get('foo')
def _commit():
t.commit()
txid = None
try:
with ZKTransaction(zkhost, timeout=0.5) as t1:
txid = t1.txid
# lock another key first to produce deadlock
t1.lock_get('woo')
t1.set_state('bar')
threadutil.start_daemon(_commit, after=0.2)
t1.lock_get('foo') # should deadlock waiting for higher txid
except Deadlock:
pass
t = ZKTransaction(zkhost)
self.assertIsNone(t.zkstorage.state.get(txid)[0])
def test_ioutil_heavy_load(self):
sess = {'running': True}
def _write(i):
while sess['running']:
with open('/tmp/pykit-test-write-' + str(i), 'w') as f:
f.write('a' * 1024*1024*10)
th1 = threadutil.start_daemon(_write, args=(1, ))
th2 = threadutil.start_daemon(_write, args=(2, ))
time.sleep(5)
force_remove('/tmp/pykit-iostat')
with ututil.Timer() as t:
rst = fsutil.iostat('/dev/sda1')
dd(rst)
self.assertAlmostEqual(100, rst['ioutil'], delta=30)
self.assertGreaterEqual(t.spent(), 1.0)
sess['running'] = False
th1.join()
th2.join()
force_remove('/tmp/pykit-test-write-1')
force_remove('/tmp/pykit-test-write-2')
def test_watch(self):
val = {'a': 2}
c = zkutil.CachedReader(self.zk, 'foo')
def _change_node():
self.zk.set('foo', utfjson.dump(val))
threadutil.start_daemon(_change_node, after=1)
self.assertEqual([self.val, val], c.watch())
def _close():
c.close()
threadutil.start_daemon(_close, after=1)
self.assertEqual(None, c.watch())
def test_watch(self):
val = {'a': 2}
c = zkutil.CachedReader(self.zk, 'foo')
def _change_node():
self.zk.set('foo', utfjson.dump(val))
threadutil.start_daemon(_change_node, after=1)
self.assertEqual([self.val, val], c.watch())
def _close():
c.close()
threadutil.start_daemon(_close, after=1)
self.assertEqual(None, c.watch())
def test_wait_absent_change_node(self):
self.zk.create('a')
change_after = 0.2
for wait_time in (
0.5,
1,
):
dd('node present wait:', wait_time)
expected = max([0, wait_time])
def _change():
time.sleep(change_after)
self.zk.set('a', 'bbb')
th = threadutil.start_daemon(target=_change)
with ututil.Timer() as t:
self.assertRaises(zkutil.ZKWaitTimeout,
zkutil.wait_absent,
self.zk, 'a', timeout=wait_time)
self.assertAlmostEqual(expected, t.spent(), delta=0.1)
th.join()
self.zk.delete('a')
def test_get_next_changed(self):
cases = (
0.4,
1,
)
def _set_a():
self.zk.set('a', 'changed')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_set_a, after=0.3)
with ututil.Timer() as t:
val, zstat = zkutil.get_next(self.zk,
'a',
timeout=timeout,
version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
self.assertEqual('changed', val)
self.assertEqual(1, zstat.version)
th.join()
self.zk.delete('a')
def test_lock_get_timeout(self):
def _tx(tx):
tx.begin()
tx.lock_get('foo')
time.sleep(4)
th = threadutil.start_daemon(_tx,
args=(ZKTransaction(zkhost, txid=0), ))
with ZKTransaction(zkhost, lock_timeout=0.5) as t1:
try:
t1.lock_get('foo')
self.fail('TXTimeout expected')
except TXTimeout as e:
dd(repr(e))
with ZKTransaction(zkhost) as t2:
try:
t2.lock_get('foo', timeout=0.5)
self.fail('TXTimeout expected')
except TXTimeout as e:
dd(repr(e))
th.join()
def test_get_next_changed_but_unsatisfied(self):
cases = (
0.4,
1,
)
def _set_a():
self.zk.set('a', 'changed')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_set_a, after=0.3)
with ututil.Timer() as t:
self.assertRaises(zkutil.ZKWaitTimeout,
zkutil.get_next,
self.zk,
'a',
timeout=timeout,
version=5)
self.assertAlmostEqual(timeout, t.spent(), delta=0.2)
th.join()
self.zk.delete('a')
def test_wait_absent_change_node(self):
self.zk.create('a')
change_after = 0.2
for wait_time in (
0.5,
1,
):
dd('node present wait:', wait_time)
expected = max([0, wait_time])
def _change():
time.sleep(change_after)
self.zk.set('a', 'bbb')
th = threadutil.start_daemon(target=_change)
with ututil.Timer() as t:
self.assertRaises(zkutil.ZKWaitTimeout,
zkutil.wait_absent,
self.zk,
'a',
timeout=wait_time)
self.assertAlmostEqual(expected, t.spent(), delta=0.1)
th.join()
self.zk.delete('a')
def test_concurrent_single_record(self):
n_tx = 10
def _tx():
while True:
try:
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = foo.v or 0
foo.v += 1
t1.set(foo)
t1.commit()
return
except Deadlock as e:
dd(repr(e))
continue
for th in [threadutil.start_daemon(_tx) for i in range(n_tx)]:
th.join()
t = ZKTransaction(zkhost)
rst, ver = t.zkstorage.record.get('foo')
dd(rst)
self.assertEqual(n_tx, rst[-1])
rst, ver = t.zkstorage.journal_id_set.get()
dd(rst)
self.assertEqual(n_tx, rst[COMMITTED].length())
def test_file_change_lost_if_too_frequently(self):
# The minimal reopen interval to detect file change
# is 0.5 sec, defined in fsutil/cat.py: file_check_time_range.
# Thus the sleep must be `>=` 0.5 or there might be chance a
# file change gets lost.
#
# Thus we need a case to expose this issue.
# 2020 Jan 05 by xp
expected = [
'a' * 32,
'b' * 32,
'c' * 32,
'd' * 32,
]
rst = []
def _override():
for l in expected:
force_remove(self.fn)
append_lines(self.fn, [l])
dd('overrided: ', l)
time.sleep(0.3)
th = threadutil.start_daemon(_override)
try:
for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=2):
rst.append(l)
except fsutil.NoData:
pass
th.join()
self.assertNotEqual(expected, rst)
def test_file_change(self):
expected = [
'a' * 32,
'b' * 32,
'c' * 32,
'd' * 32,
]
rst = []
def _override():
for l in expected:
force_remove(self.fn)
append_lines(self.fn, [l])
dd('overrided: ', l)
time.sleep(0.6)
th = threadutil.start_daemon(_override)
try:
for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=2):
rst.append(l)
except fsutil.NoData:
pass
th.join()
self.assertEqual(expected, rst)
def test_wait_for_data_timeout(self):
expected = [
'a' * 32,
'b' * 32,
]
rst = []
append_lines(self.fn, expected)
dd('appended 1')
def _append():
time.sleep(2)
append_lines(self.fn, expected)
dd('appended')
th = threadutil.start_daemon(_append)
try:
for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=0.1):
rst.append(l)
self.fail('expect NoData to raise')
except fsutil.NoData:
pass
th.join()
self.assertEqual(expected, rst)
def test_wait_for_file_timeout(self):
expected = [
'a' * 32,
'b' * 32,
]
rst = []
def _append():
time.sleep(0.3)
append_lines(self.fn, expected)
dd('appended')
th = threadutil.start_daemon(_append)
try:
for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=0.1):
rst.append(l)
self.fail('expect NoSuchFile to raise')
except fsutil.NoSuchFile:
pass
self.assertEqual([], rst)
th.join()
def test_lock_get_timeout(self):
def _tx(tx):
tx.begin()
tx.lock_get('foo')
time.sleep(4)
th = threadutil.start_daemon(_tx, args=(ZKTransaction(zkhost, txid=0),))
with ZKTransaction(zkhost, lock_timeout=0.5) as t1:
try:
t1.lock_get('foo')
self.fail('TXTimeout expected')
except TXTimeout as e:
dd(repr(e))
with ZKTransaction(zkhost) as t2:
try:
t2.lock_get('foo', timeout=0.5)
self.fail('TXTimeout expected')
except TXTimeout as e:
dd(repr(e))
th.join()
def test_concurrent_single_record(self):
n_tx = 10
def _tx():
while True:
try:
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = foo.v or 0
foo.v += 1
t1.set(foo)
t1.commit()
return
except Deadlock as e:
dd(repr(e))
continue
for th in [threadutil.start_daemon(_tx)
for i in range(n_tx)]:
th.join()
t = ZKTransaction(zkhost)
rst, ver = t.zkstorage.record.get('foo')
dd(rst)
self.assertEqual(n_tx, rst[-1])
rst, ver = t.zkstorage.journal_id_set.get()
dd(rst)
self.assertEqual(n_tx, rst[COMMITTED].length())
def test_get_next_deleted(self):
cases = (
0.4,
1,
)
def _del_a():
self.zk.delete('a')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_del_a, after=0.3)
with ututil.Timer() as t:
self.assertRaises(NoNodeError,
zkutil.get_next,
self.zk,
'a',
timeout=timeout,
version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
th.join()
def test_concurrent_3_record(self):
n_tx = 10
ks = ('foo', 'bar', 'duu')
def _tx(i):
while True:
try:
with ZKTransaction(zkhost) as t1:
for ii in range(len(ks)):
k = ks[(ii+i) % len(ks)]
foo = t1.lock_get(k)
foo.v = foo.v or 0
foo.v += 1
t1.set(foo)
t1.commit()
dd(str(t1) + ' committed')
return
except (Deadlock,
HigherTXApplied) as e:
dd(str(t1) + ': ' + repr(e))
continue
except TXTimeout as e:
dd(str(t1) + ': ' + repr(e))
raise
with ututil.Timer() as tt:
for th in [threadutil.start_daemon(_tx, args=(i, ))
for i in range(n_tx)]:
th.join()
dd('3 key 10 thread: ', tt.spent())
t = ZKTransaction(zkhost)
for k in ks:
rst, ver = t.zkstorage.record.get(k)
dd(rst)
self.assertEqual(n_tx, rst[-1][1])
rst, ver = t.zkstorage.txidset.get()
dd(rst)
self.assertEqual(n_tx, rst[COMMITTED].length())
# check aborted txidset
sm = rangeset.union(rst[COMMITTED], rst[PURGED])
self.assertEqual(rst[COMMITTED][-1][-1] - 1, sm.length())
def test_get_next_conn_lost(self):
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=self.zk.stop, after=0.3)
with ututil.Timer() as t:
self.assertRaises(ConnectionClosedError,
zkutil.get_next,
self.zk, 'a', timeout=1, version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
th.join()
def test_cas_concurrent(self):
def _update():
for ii in range(10):
for curr in txutil.cas_loop(self._get, self._set):
curr.v += 1
ths = [threadutil.start_daemon(_update) for _ in range(10)]
for th in ths:
th.join()
self.assertEqual((100, 100), (self.val, self.ver))
def test_file_end_handler(self):
expected = [
'a' * 32,
'b' * 32,
]
append_lines(self.fn, expected)
rst = []
def _end():
rst.append('end')
# file_end_handler in cat()
c = fsutil.Cat(self.fn,
strip=True,
handler=rst.append,
file_end_handler=_end)
c.cat(timeout=0)
self.assertEqual(expected + ['end'], rst)
force_remove(c.stat_path())
# file_end_handler in iterate()
rst = []
force_remove(self.fn)
append_lines(self.fn, expected)
for line in fsutil.Cat(self.fn, strip=True,
file_end_handler=_end).iterate(timeout=0):
rst.append(line)
self.assertEqual(expected + ['end'], rst)
force_remove(c.stat_path())
# file_end_handler multi times
rst = []
force_remove(self.fn)
append_lines(self.fn, expected)
def _append():
time.sleep(1)
append_lines(self.fn, expected)
th = threadutil.start_daemon(_append)
try:
for line in fsutil.Cat(self.fn, strip=True,
file_end_handler=_end).iterate(timeout=0.2):
rst.append(line)
except fsutil.NoData:
pass
th.join()
self.assertEqual((expected + ['end']) * 2, rst)
def test_conn_lost_when_blocking_acquiring(self):
l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True)
th = threadutil.start_daemon(target=self.zk.stop, after=0.5)
with l2:
try:
self.lck.acquire(timeout=1)
self.fail('expected connection error')
except ConnectionClosedError:
pass
th.join()
def test_cas_concurrent(self):
def _update():
for ii in range(10):
for curr in txutil.cas_loop(self._get, self._set):
curr.v += 1
ths = [threadutil.start_daemon(_update)
for _ in range(10)]
for th in ths:
th.join()
self.assertEqual((100, 100), (self.val, self.ver))
def test_concurrent_3_record(self):
n_tx = 10
ks = ('foo', 'bar', 'duu')
def _tx(i):
while True:
try:
with ZKTransaction(zkhost) as t1:
for ii in range(len(ks)):
k = ks[(ii+i) % len(ks)]
foo = t1.lock_get(k)
foo.v = foo.v or 0
foo.v += 1
t1.set(foo)
t1.commit()
dd(str(t1) + ' committed')
return
except Deadlock as e:
dd(str(t1) + ': ' + repr(e))
continue
except TXTimeout as e:
dd(str(t1) + ': ' + repr(e))
raise
with ututil.Timer() as tt:
for th in [threadutil.start_daemon(_tx, args=(i, ))
for i in range(n_tx)]:
th.join()
dd('3 key 10 thread: ', tt.spent())
t = ZKTransaction(zkhost)
for k in ks:
rst, ver = t.zkstorage.record.get(k)
dd(rst)
self.assertEqual(n_tx, rst[-1])
rst, ver = t.zkstorage.journal_id_set.get()
dd(rst)
self.assertEqual(n_tx, rst[COMMITTED].length())
def test_cas_concurrent(self):
def _update():
for ii in range(10):
for curr in zkutil.cas_loop('127.0.0.1:21811', self.path):
curr.v += 1
ths = [threadutil.start_daemon(_update) for _ in range(5)]
for th in ths:
th.join()
final_val, zstat = self.zk.get(self.path)
dd(final_val, zstat)
self.assertEqual('51', final_val)
def test_get_next_conn_lost(self):
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=self.zk.stop, after=0.3)
with ututil.Timer() as t:
self.assertRaises(ConnectionClosedError,
zkutil.get_next,
self.zk,
'a',
timeout=1,
version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
th.join()
def test_wait_absent_connection_lost(self):
self.zk.create('a')
def _close():
time.sleep(.3)
self.zk.stop()
th = threadutil.start_daemon(target=_close)
with ututil.Timer() as t:
self.assertRaises(ConnectionClosedError, zkutil.wait_absent,
self.zk, 'a')
self.assertAlmostEqual(.3, t.spent(), delta=0.1)
th.join()
def test_cas_concurrent(self):
def _update():
for ii in range(10):
for curr in zkutil.cas_loop('127.0.0.1:21811', self.path):
curr.v += 1
ths = [threadutil.start_daemon(_update)
for _ in range(5)]
for th in ths:
th.join()
final_val, zstat = self.zk.get(self.path)
dd(final_val, zstat)
self.assertEqual('51', final_val)
def test_wait_absent_connection_lost(self):
self.zk.create('a')
def _close():
time.sleep(.3)
self.zk.stop()
th = threadutil.start_daemon(target=_close)
with ututil.Timer() as t:
self.assertRaises(ConnectionClosedError,
zkutil.wait_absent,
self.zk, 'a')
self.assertAlmostEqual(.3, t.spent(), delta=0.1)
th.join()
def test_wait_absent_no_timeout(self):
def _del():
time.sleep(1)
self.zk.delete('a')
for kwargs in (
{},
{'timeout': None},
):
self.zk.create('a')
th = threadutil.start_daemon(target=_del)
with ututil.Timer() as t:
zkutil.wait_absent(self.zk, 'a', **kwargs)
self.assertAlmostEqual(1, t.spent(), delta=0.1)
th.join()
def test_wait_absent_no_timeout(self):
def _del():
time.sleep(1)
self.zk.delete('a')
for kwargs in (
{},
{
'timeout': None
},
):
self.zk.create('a')
th = threadutil.start_daemon(target=_del)
with ututil.Timer() as t:
zkutil.wait_absent(self.zk, 'a', **kwargs)
self.assertAlmostEqual(1, t.spent(), delta=0.1)
th.join()
def test_run_tx_lock_timeout(self):
def _tx0(tx):
tx.begin()
tx.lock_get('foo')
time.sleep(2)
def _tx1(tx):
tx.lock_get('foo')
time.sleep(0.2)
tx.commit()
th = threadutil.start_daemon(_tx0, args=(ZKTransaction(zkhost, txid=0),))
try:
zktx.run_tx(zkhost, _tx1, lock_timeout=0.4)
self.fail('TXTimeout expected')
except TXTimeout as e:
dd(repr(e))
th.join()
def test_run_tx_lock_timeout(self):
def _tx0(tx):
tx.begin()
tx.lock_get('foo')
time.sleep(2)
def _tx1(tx):
tx.lock_get('foo')
time.sleep(0.2)
tx.commit()
th = threadutil.start_daemon(_tx0,
args=(ZKTransaction(zkhost, txid=0), ))
try:
zktx.run_tx(zkhost, _tx1, lock_timeout=0.4)
self.fail('TXTimeout expected')
except TXTimeout as e:
dd(repr(e))
th.join()
def test_wait_absent_delete_node(self):
delete_after = 0.2
for wait_time in (
0.5,
1,
):
dd('node present wait:', wait_time)
self.zk.create('a')
def _del():
time.sleep(delete_after)
self.zk.delete('a')
th = threadutil.start_daemon(target=_del)
with ututil.Timer() as t:
zkutil.wait_absent(self.zk, 'a', wait_time)
self.assertAlmostEqual(delete_after, t.spent(), delta=0.1)
th.join()
def test_get_next_deleted(self):
cases = (
0.4,
1,
)
def _del_a():
self.zk.delete('a')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_del_a, after=0.3)
with ututil.Timer() as t:
self.assertRaises(NoNodeError,
zkutil.get_next,
self.zk, 'a', timeout=timeout, version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
th.join()
def test_wait_absent_delete_node(self):
delete_after = 0.2
for wait_time in (
0.5,
1,
):
dd('node present wait:', wait_time)
self.zk.create('a')
def _del():
time.sleep(delete_after)
self.zk.delete('a')
th = threadutil.start_daemon(target=_del)
with ututil.Timer() as t:
zkutil.wait_absent(self.zk, 'a', wait_time)
self.assertAlmostEqual(delete_after, t.spent(), delta=0.1)
th.join()
def test_get_next_changed_but_unsatisfied(self):
cases = (
0.4,
1,
)
def _set_a():
self.zk.set('a', 'changed')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_set_a, after=0.3)
with ututil.Timer() as t:
self.assertRaises(zkutil.ZKWaitTimeout,
zkutil.get_next,
self.zk, 'a', timeout=timeout, version=5)
self.assertAlmostEqual(timeout, t.spent(), delta=0.2)
th.join()
self.zk.delete('a')
def test_get_next_changed(self):
cases = (
0.4,
1,
)
def _set_a():
self.zk.set('a', 'changed')
for timeout in cases:
self.zk.create('a', 'a-val')
th = threadutil.start_daemon(target=_set_a, after=0.3)
with ututil.Timer() as t:
val, zstat = zkutil.get_next(
self.zk, 'a', timeout=timeout, version=0)
self.assertAlmostEqual(0.3, t.spent(), delta=0.2)
self.assertEqual('changed', val)
self.assertEqual(1, zstat.version)
th.join()
self.zk.delete('a')
def test_half_line(self):
n = 128
def _append():
for _ in range(n):
time.sleep(0.001)
append_bytes(self.fn, 'a' * 2048)
time.sleep(0.001)
append_bytes(self.fn, 'b' * 2048 + '\n')
th = threadutil.start_daemon(_append)
i = 0
try:
for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=0.1):
self.assertEqual('b', l[-1])
i += 1
except fsutil.NoData:
pass
th.join()
self.assertEqual(n, i)
