def test_basic(self): log_entries = [] def send_log(log_entry): log_entries.append(log_entry) kwargs = { 'node_id': '123abc', 'node_ip': '1.2.3.4', 'send_log': send_log, 'conf': { 'my_test_log': { 'file_path': os.path.join(this_base, 'test_log.out'), 'level': ['error'], 'get_level': self.get_level, 'is_first_line': is_first_line, 'parse': parse, }, }, } threadutil.start_daemon(self.log) threadutil.start_daemon(collector.run, kwargs=kwargs) time.sleep(8) dd(log_entries) log_cnt = 0 for le in log_entries: log_cnt += le["count"] dd(log_cnt) self.assertEqual(100, log_cnt) self.assertEqual('error', log_entries[0]['level']) self.assertEqual('my_test_log', log_entries[0]['log_name']) self.assertEqual('test_log.out', log_entries[0]['log_file'])
def test_no_merge(self): log_entries = [] def send_log(log_entry): print log_entry log_entries.append(log_entry) kwargs = { 'node_id': '123abc', 'node_ip': '1.2.3.4', 'send_log': send_log, 'conf': { 'my_test_log': { 'file_path': os.path.join(this_base, 'test_log.out'), 'level': ['error'], 'get_level': self.get_level, 'is_first_line': is_first_line, 'parse': parse, 'merge': False, }, }, } threadutil.start_daemon(self.log) threadutil.start_daemon(collector.run, kwargs=kwargs) time.sleep(8) self.assertEqual(100, len(log_entries)) self.assertEqual('error', log_entries[0]['level'])
def test_deadlock(self): # deadlock is not recoverable t = ZKTransaction(zkhost) # t.txid is lower t.begin() t.lock_get('foo') def _commit(): t.commit() txid = None try: with ZKTransaction(zkhost, timeout=0.5) as t1: txid = t1.txid # lock another key first to produce deadlock t1.lock_get('woo') t1.set_state('bar') threadutil.start_daemon(_commit, after=0.2) t1.lock_get('foo') # should deadlock waiting for higher txid except Deadlock: pass t = ZKTransaction(zkhost) self.assertIsNone(t.zkstorage.state.get(txid)[0])
def test_ioutil_heavy_load(self): sess = {'running': True} def _write(i): while sess['running']: with open('/tmp/pykit-test-write-' + str(i), 'w') as f: f.write('a' * 1024*1024*10) th1 = threadutil.start_daemon(_write, args=(1, )) th2 = threadutil.start_daemon(_write, args=(2, )) time.sleep(5) force_remove('/tmp/pykit-iostat') with ututil.Timer() as t: rst = fsutil.iostat('/dev/sda1') dd(rst) self.assertAlmostEqual(100, rst['ioutil'], delta=40) self.assertGreaterEqual(t.spent(), 1.0) sess['running'] = False th1.join() th2.join() force_remove('/tmp/pykit-test-write-1') force_remove('/tmp/pykit-test-write-2')
def test_deadlock(self): # deadlock is not recoverable t = ZKTransaction(zkhost) # t.txid is lower t.begin() t.lock_get('foo') def _commit(): t.commit() txid = None try: with ZKTransaction(zkhost, timeout=0.5) as t1: txid = t1.txid # lock another key first to produce deadlock t1.lock_get('woo') t1.set_state('bar') threadutil.start_daemon(_commit, after=0.2) t1.lock_get('foo') # should deadlock waiting for higher txid except Deadlock: pass t = ZKTransaction(zkhost) self.assertIsNone(t.zkstorage.state.get(txid)[0])
def test_ioutil_heavy_load(self): sess = {'running': True} def _write(i): while sess['running']: with open('/tmp/pykit-test-write-' + str(i), 'w') as f: f.write('a' * 1024*1024*10) th1 = threadutil.start_daemon(_write, args=(1, )) th2 = threadutil.start_daemon(_write, args=(2, )) time.sleep(5) force_remove('/tmp/pykit-iostat') with ututil.Timer() as t: rst = fsutil.iostat('/dev/sda1') dd(rst) self.assertAlmostEqual(100, rst['ioutil'], delta=30) self.assertGreaterEqual(t.spent(), 1.0) sess['running'] = False th1.join() th2.join() force_remove('/tmp/pykit-test-write-1') force_remove('/tmp/pykit-test-write-2')
def test_watch(self): val = {'a': 2} c = zkutil.CachedReader(self.zk, 'foo') def _change_node(): self.zk.set('foo', utfjson.dump(val)) threadutil.start_daemon(_change_node, after=1) self.assertEqual([self.val, val], c.watch()) def _close(): c.close() threadutil.start_daemon(_close, after=1) self.assertEqual(None, c.watch())
def test_watch(self): val = {'a': 2} c = zkutil.CachedReader(self.zk, 'foo') def _change_node(): self.zk.set('foo', utfjson.dump(val)) threadutil.start_daemon(_change_node, after=1) self.assertEqual([self.val, val], c.watch()) def _close(): c.close() threadutil.start_daemon(_close, after=1) self.assertEqual(None, c.watch())
def test_wait_absent_change_node(self): self.zk.create('a') change_after = 0.2 for wait_time in ( 0.5, 1, ): dd('node present wait:', wait_time) expected = max([0, wait_time]) def _change(): time.sleep(change_after) self.zk.set('a', 'bbb') th = threadutil.start_daemon(target=_change) with ututil.Timer() as t: self.assertRaises(zkutil.ZKWaitTimeout, zkutil.wait_absent, self.zk, 'a', timeout=wait_time) self.assertAlmostEqual(expected, t.spent(), delta=0.1) th.join() self.zk.delete('a')
def test_get_next_changed(self): cases = ( 0.4, 1, ) def _set_a(): self.zk.set('a', 'changed') for timeout in cases: self.zk.create('a', 'a-val') th = threadutil.start_daemon(target=_set_a, after=0.3) with ututil.Timer() as t: val, zstat = zkutil.get_next(self.zk, 'a', timeout=timeout, version=0) self.assertAlmostEqual(0.3, t.spent(), delta=0.2) self.assertEqual('changed', val) self.assertEqual(1, zstat.version) th.join() self.zk.delete('a')
def test_lock_get_timeout(self): def _tx(tx): tx.begin() tx.lock_get('foo') time.sleep(4) th = threadutil.start_daemon(_tx, args=(ZKTransaction(zkhost, txid=0), )) with ZKTransaction(zkhost, lock_timeout=0.5) as t1: try: t1.lock_get('foo') self.fail('TXTimeout expected') except TXTimeout as e: dd(repr(e)) with ZKTransaction(zkhost) as t2: try: t2.lock_get('foo', timeout=0.5) self.fail('TXTimeout expected') except TXTimeout as e: dd(repr(e)) th.join()
def test_get_next_changed_but_unsatisfied(self): cases = ( 0.4, 1, ) def _set_a(): self.zk.set('a', 'changed') for timeout in cases: self.zk.create('a', 'a-val') th = threadutil.start_daemon(target=_set_a, after=0.3) with ututil.Timer() as t: self.assertRaises(zkutil.ZKWaitTimeout, zkutil.get_next, self.zk, 'a', timeout=timeout, version=5) self.assertAlmostEqual(timeout, t.spent(), delta=0.2) th.join() self.zk.delete('a')
def test_wait_absent_change_node(self): self.zk.create('a') change_after = 0.2 for wait_time in ( 0.5, 1, ): dd('node present wait:', wait_time) expected = max([0, wait_time]) def _change(): time.sleep(change_after) self.zk.set('a', 'bbb') th = threadutil.start_daemon(target=_change) with ututil.Timer() as t: self.assertRaises(zkutil.ZKWaitTimeout, zkutil.wait_absent, self.zk, 'a', timeout=wait_time) self.assertAlmostEqual(expected, t.spent(), delta=0.1) th.join() self.zk.delete('a')
def test_concurrent_single_record(self): n_tx = 10 def _tx(): while True: try: with ZKTransaction(zkhost) as t1: foo = t1.lock_get('foo') foo.v = foo.v or 0 foo.v += 1 t1.set(foo) t1.commit() return except Deadlock as e: dd(repr(e)) continue for th in [threadutil.start_daemon(_tx) for i in range(n_tx)]: th.join() t = ZKTransaction(zkhost) rst, ver = t.zkstorage.record.get('foo') dd(rst) self.assertEqual(n_tx, rst[-1]) rst, ver = t.zkstorage.journal_id_set.get() dd(rst) self.assertEqual(n_tx, rst[COMMITTED].length())
def test_file_change_lost_if_too_frequently(self): # The minimal reopen interval to detect file change # is 0.5 sec, defined in fsutil/cat.py: file_check_time_range. # Thus the sleep must be `>=` 0.5 or there might be chance a # file change gets lost. # # Thus we need a case to expose this issue. # 2020 Jan 05 by xp expected = [ 'a' * 32, 'b' * 32, 'c' * 32, 'd' * 32, ] rst = [] def _override(): for l in expected: force_remove(self.fn) append_lines(self.fn, [l]) dd('overrided: ', l) time.sleep(0.3) th = threadutil.start_daemon(_override) try: for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=2): rst.append(l) except fsutil.NoData: pass th.join() self.assertNotEqual(expected, rst)
def test_file_change(self): expected = [ 'a' * 32, 'b' * 32, 'c' * 32, 'd' * 32, ] rst = [] def _override(): for l in expected: force_remove(self.fn) append_lines(self.fn, [l]) dd('overrided: ', l) time.sleep(0.6) th = threadutil.start_daemon(_override) try: for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=2): rst.append(l) except fsutil.NoData: pass th.join() self.assertEqual(expected, rst)
def test_wait_for_data_timeout(self): expected = [ 'a' * 32, 'b' * 32, ] rst = [] append_lines(self.fn, expected) dd('appended 1') def _append(): time.sleep(2) append_lines(self.fn, expected) dd('appended') th = threadutil.start_daemon(_append) try: for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=0.1): rst.append(l) self.fail('expect NoData to raise') except fsutil.NoData: pass th.join() self.assertEqual(expected, rst)
def test_wait_for_file_timeout(self): expected = [ 'a' * 32, 'b' * 32, ] rst = [] def _append(): time.sleep(0.3) append_lines(self.fn, expected) dd('appended') th = threadutil.start_daemon(_append) try: for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=0.1): rst.append(l) self.fail('expect NoSuchFile to raise') except fsutil.NoSuchFile: pass self.assertEqual([], rst) th.join()
def test_lock_get_timeout(self): def _tx(tx): tx.begin() tx.lock_get('foo') time.sleep(4) th = threadutil.start_daemon(_tx, args=(ZKTransaction(zkhost, txid=0),)) with ZKTransaction(zkhost, lock_timeout=0.5) as t1: try: t1.lock_get('foo') self.fail('TXTimeout expected') except TXTimeout as e: dd(repr(e)) with ZKTransaction(zkhost) as t2: try: t2.lock_get('foo', timeout=0.5) self.fail('TXTimeout expected') except TXTimeout as e: dd(repr(e)) th.join()
def test_concurrent_single_record(self): n_tx = 10 def _tx(): while True: try: with ZKTransaction(zkhost) as t1: foo = t1.lock_get('foo') foo.v = foo.v or 0 foo.v += 1 t1.set(foo) t1.commit() return except Deadlock as e: dd(repr(e)) continue for th in [threadutil.start_daemon(_tx) for i in range(n_tx)]: th.join() t = ZKTransaction(zkhost) rst, ver = t.zkstorage.record.get('foo') dd(rst) self.assertEqual(n_tx, rst[-1]) rst, ver = t.zkstorage.journal_id_set.get() dd(rst) self.assertEqual(n_tx, rst[COMMITTED].length())
def test_get_next_deleted(self): cases = ( 0.4, 1, ) def _del_a(): self.zk.delete('a') for timeout in cases: self.zk.create('a', 'a-val') th = threadutil.start_daemon(target=_del_a, after=0.3) with ututil.Timer() as t: self.assertRaises(NoNodeError, zkutil.get_next, self.zk, 'a', timeout=timeout, version=0) self.assertAlmostEqual(0.3, t.spent(), delta=0.2) th.join()
def test_concurrent_3_record(self): n_tx = 10 ks = ('foo', 'bar', 'duu') def _tx(i): while True: try: with ZKTransaction(zkhost) as t1: for ii in range(len(ks)): k = ks[(ii+i) % len(ks)] foo = t1.lock_get(k) foo.v = foo.v or 0 foo.v += 1 t1.set(foo) t1.commit() dd(str(t1) + ' committed') return except (Deadlock, HigherTXApplied) as e: dd(str(t1) + ': ' + repr(e)) continue except TXTimeout as e: dd(str(t1) + ': ' + repr(e)) raise with ututil.Timer() as tt: for th in [threadutil.start_daemon(_tx, args=(i, )) for i in range(n_tx)]: th.join() dd('3 key 10 thread: ', tt.spent()) t = ZKTransaction(zkhost) for k in ks: rst, ver = t.zkstorage.record.get(k) dd(rst) self.assertEqual(n_tx, rst[-1][1]) rst, ver = t.zkstorage.txidset.get() dd(rst) self.assertEqual(n_tx, rst[COMMITTED].length()) # check aborted txidset sm = rangeset.union(rst[COMMITTED], rst[PURGED]) self.assertEqual(rst[COMMITTED][-1][-1] - 1, sm.length())
def test_get_next_conn_lost(self): self.zk.create('a', 'a-val') th = threadutil.start_daemon(target=self.zk.stop, after=0.3) with ututil.Timer() as t: self.assertRaises(ConnectionClosedError, zkutil.get_next, self.zk, 'a', timeout=1, version=0) self.assertAlmostEqual(0.3, t.spent(), delta=0.2) th.join()
def test_cas_concurrent(self): def _update(): for ii in range(10): for curr in txutil.cas_loop(self._get, self._set): curr.v += 1 ths = [threadutil.start_daemon(_update) for _ in range(10)] for th in ths: th.join() self.assertEqual((100, 100), (self.val, self.ver))
def test_file_end_handler(self): expected = [ 'a' * 32, 'b' * 32, ] append_lines(self.fn, expected) rst = [] def _end(): rst.append('end') # file_end_handler in cat() c = fsutil.Cat(self.fn, strip=True, handler=rst.append, file_end_handler=_end) c.cat(timeout=0) self.assertEqual(expected + ['end'], rst) force_remove(c.stat_path()) # file_end_handler in iterate() rst = [] force_remove(self.fn) append_lines(self.fn, expected) for line in fsutil.Cat(self.fn, strip=True, file_end_handler=_end).iterate(timeout=0): rst.append(line) self.assertEqual(expected + ['end'], rst) force_remove(c.stat_path()) # file_end_handler multi times rst = [] force_remove(self.fn) append_lines(self.fn, expected) def _append(): time.sleep(1) append_lines(self.fn, expected) th = threadutil.start_daemon(_append) try: for line in fsutil.Cat(self.fn, strip=True, file_end_handler=_end).iterate(timeout=0.2): rst.append(line) except fsutil.NoData: pass th.join() self.assertEqual((expected + ['end']) * 2, rst)
def test_conn_lost_when_blocking_acquiring(self): l2 = zkutil.ZKLock('foo_name', on_lost=lambda: True) th = threadutil.start_daemon(target=self.zk.stop, after=0.5) with l2: try: self.lck.acquire(timeout=1) self.fail('expected connection error') except ConnectionClosedError: pass th.join()
def test_cas_concurrent(self): def _update(): for ii in range(10): for curr in txutil.cas_loop(self._get, self._set): curr.v += 1 ths = [threadutil.start_daemon(_update) for _ in range(10)] for th in ths: th.join() self.assertEqual((100, 100), (self.val, self.ver))
def test_concurrent_3_record(self): n_tx = 10 ks = ('foo', 'bar', 'duu') def _tx(i): while True: try: with ZKTransaction(zkhost) as t1: for ii in range(len(ks)): k = ks[(ii+i) % len(ks)] foo = t1.lock_get(k) foo.v = foo.v or 0 foo.v += 1 t1.set(foo) t1.commit() dd(str(t1) + ' committed') return except Deadlock as e: dd(str(t1) + ': ' + repr(e)) continue except TXTimeout as e: dd(str(t1) + ': ' + repr(e)) raise with ututil.Timer() as tt: for th in [threadutil.start_daemon(_tx, args=(i, )) for i in range(n_tx)]: th.join() dd('3 key 10 thread: ', tt.spent()) t = ZKTransaction(zkhost) for k in ks: rst, ver = t.zkstorage.record.get(k) dd(rst) self.assertEqual(n_tx, rst[-1]) rst, ver = t.zkstorage.journal_id_set.get() dd(rst) self.assertEqual(n_tx, rst[COMMITTED].length())
def test_cas_concurrent(self): def _update(): for ii in range(10): for curr in zkutil.cas_loop('127.0.0.1:21811', self.path): curr.v += 1 ths = [threadutil.start_daemon(_update) for _ in range(5)] for th in ths: th.join() final_val, zstat = self.zk.get(self.path) dd(final_val, zstat) self.assertEqual('51', final_val)
def test_get_next_conn_lost(self): self.zk.create('a', 'a-val') th = threadutil.start_daemon(target=self.zk.stop, after=0.3) with ututil.Timer() as t: self.assertRaises(ConnectionClosedError, zkutil.get_next, self.zk, 'a', timeout=1, version=0) self.assertAlmostEqual(0.3, t.spent(), delta=0.2) th.join()
def test_wait_absent_connection_lost(self): self.zk.create('a') def _close(): time.sleep(.3) self.zk.stop() th = threadutil.start_daemon(target=_close) with ututil.Timer() as t: self.assertRaises(ConnectionClosedError, zkutil.wait_absent, self.zk, 'a') self.assertAlmostEqual(.3, t.spent(), delta=0.1) th.join()
def test_cas_concurrent(self): def _update(): for ii in range(10): for curr in zkutil.cas_loop('127.0.0.1:21811', self.path): curr.v += 1 ths = [threadutil.start_daemon(_update) for _ in range(5)] for th in ths: th.join() final_val, zstat = self.zk.get(self.path) dd(final_val, zstat) self.assertEqual('51', final_val)
def test_wait_absent_connection_lost(self): self.zk.create('a') def _close(): time.sleep(.3) self.zk.stop() th = threadutil.start_daemon(target=_close) with ututil.Timer() as t: self.assertRaises(ConnectionClosedError, zkutil.wait_absent, self.zk, 'a') self.assertAlmostEqual(.3, t.spent(), delta=0.1) th.join()
def test_wait_absent_no_timeout(self): def _del(): time.sleep(1) self.zk.delete('a') for kwargs in ( {}, {'timeout': None}, ): self.zk.create('a') th = threadutil.start_daemon(target=_del) with ututil.Timer() as t: zkutil.wait_absent(self.zk, 'a', **kwargs) self.assertAlmostEqual(1, t.spent(), delta=0.1) th.join()
def test_wait_absent_no_timeout(self): def _del(): time.sleep(1) self.zk.delete('a') for kwargs in ( {}, { 'timeout': None }, ): self.zk.create('a') th = threadutil.start_daemon(target=_del) with ututil.Timer() as t: zkutil.wait_absent(self.zk, 'a', **kwargs) self.assertAlmostEqual(1, t.spent(), delta=0.1) th.join()
def test_run_tx_lock_timeout(self): def _tx0(tx): tx.begin() tx.lock_get('foo') time.sleep(2) def _tx1(tx): tx.lock_get('foo') time.sleep(0.2) tx.commit() th = threadutil.start_daemon(_tx0, args=(ZKTransaction(zkhost, txid=0),)) try: zktx.run_tx(zkhost, _tx1, lock_timeout=0.4) self.fail('TXTimeout expected') except TXTimeout as e: dd(repr(e)) th.join()
def test_run_tx_lock_timeout(self): def _tx0(tx): tx.begin() tx.lock_get('foo') time.sleep(2) def _tx1(tx): tx.lock_get('foo') time.sleep(0.2) tx.commit() th = threadutil.start_daemon(_tx0, args=(ZKTransaction(zkhost, txid=0), )) try: zktx.run_tx(zkhost, _tx1, lock_timeout=0.4) self.fail('TXTimeout expected') except TXTimeout as e: dd(repr(e)) th.join()
def test_wait_absent_delete_node(self): delete_after = 0.2 for wait_time in ( 0.5, 1, ): dd('node present wait:', wait_time) self.zk.create('a') def _del(): time.sleep(delete_after) self.zk.delete('a') th = threadutil.start_daemon(target=_del) with ututil.Timer() as t: zkutil.wait_absent(self.zk, 'a', wait_time) self.assertAlmostEqual(delete_after, t.spent(), delta=0.1) th.join()
def test_get_next_deleted(self): cases = ( 0.4, 1, ) def _del_a(): self.zk.delete('a') for timeout in cases: self.zk.create('a', 'a-val') th = threadutil.start_daemon(target=_del_a, after=0.3) with ututil.Timer() as t: self.assertRaises(NoNodeError, zkutil.get_next, self.zk, 'a', timeout=timeout, version=0) self.assertAlmostEqual(0.3, t.spent(), delta=0.2) th.join()
def test_wait_absent_delete_node(self): delete_after = 0.2 for wait_time in ( 0.5, 1, ): dd('node present wait:', wait_time) self.zk.create('a') def _del(): time.sleep(delete_after) self.zk.delete('a') th = threadutil.start_daemon(target=_del) with ututil.Timer() as t: zkutil.wait_absent(self.zk, 'a', wait_time) self.assertAlmostEqual(delete_after, t.spent(), delta=0.1) th.join()
def test_get_next_changed_but_unsatisfied(self): cases = ( 0.4, 1, ) def _set_a(): self.zk.set('a', 'changed') for timeout in cases: self.zk.create('a', 'a-val') th = threadutil.start_daemon(target=_set_a, after=0.3) with ututil.Timer() as t: self.assertRaises(zkutil.ZKWaitTimeout, zkutil.get_next, self.zk, 'a', timeout=timeout, version=5) self.assertAlmostEqual(timeout, t.spent(), delta=0.2) th.join() self.zk.delete('a')
def test_get_next_changed(self): cases = ( 0.4, 1, ) def _set_a(): self.zk.set('a', 'changed') for timeout in cases: self.zk.create('a', 'a-val') th = threadutil.start_daemon(target=_set_a, after=0.3) with ututil.Timer() as t: val, zstat = zkutil.get_next( self.zk, 'a', timeout=timeout, version=0) self.assertAlmostEqual(0.3, t.spent(), delta=0.2) self.assertEqual('changed', val) self.assertEqual(1, zstat.version) th.join() self.zk.delete('a')
def test_half_line(self): n = 128 def _append(): for _ in range(n): time.sleep(0.001) append_bytes(self.fn, 'a' * 2048) time.sleep(0.001) append_bytes(self.fn, 'b' * 2048 + '\n') th = threadutil.start_daemon(_append) i = 0 try: for l in fsutil.Cat(self.fn, strip=True).iterate(timeout=0.1): self.assertEqual('b', l[-1]) i += 1 except fsutil.NoData: pass th.join() self.assertEqual(n, i)