def __init__(self,
zk,
txid=None,
timeout=DEFAULT_TIMEOUT,
lock_timeout=None):
# Save the original arg for self.run()
self._zk = zk
self.zke, self.owning_zk = zkutil.kazoo_client_ext(zk)
self.timeout = timeout
self.lock_timeout = lock_timeout
self.txid = txid
self.expire_at = None
self.tx_alive_lock = None
# User locked and required keys
self.got_keys = {}
# Keys user need to commit
self.modifications = {}
self.state_lock = threading.RLock()
self.connected = True
self.tx_status = None
self.zkstorage = ZKStorage(self.zke)
self.zkstorage.max_value_history = 2
self.zke.add_listener(self._on_conn_change)
def setUp(self):
super(TestSlave, self).setUp()
utdocker.create_network()
utdocker.start_container('redis-0', redis_tag, '192.168.52.40', '')
dd('started redis in docker')
redisutil.wait_serve(('192.168.52.40', redis_port))
self.redis_cli = redisutil.get_client(('192.168.52.40', redis_port))
self.journal_id_path = "tx/journal_id_set"
self.storage = zktx.RedisStorage(self.redis_cli, self.journal_id_path)
self.zk.create('tx', 'tx')
self.zk.create('tx/journal_id_set', '{}')
self.zk.create('tx/alive', '{}')
self.zk.create('tx/txid_maker', '{}')
self.zk.create('tx/journal', '{}')
for d in ('lock', 'record'):
self.zk.create(d, '{}')
for d1 in ('leader', 'meta', 'job'):
self.zk.create('/'.join([d, d1]), '{}')
for d2 in ('block_group', 'global', 'region', 'server'):
self.zk.create('/'.join([d, d1, d2]), '{}')
self.zke, _ = zkutil.kazoo_client_ext(self.zk)
self.slave = zktx.Slave(self.zke, self.storage)
def _set_all_records(self):
record_dir = self.zke._zkconf.record_dir()
meta_path = ''.join([record_dir, 'meta'])
names = self.zke.get_children(meta_path)
lock = threading.RLock()
st = {'existed': {}, 'failed': []}
zke_pool = cachepool.CachePool(lambda c: zkutil.kazoo_client_ext(c)[0],
generator_args=[self.zke._zkconf],
close_callback=zkutil.close_zk)
def _iter_key():
for n in names:
keys_path = '/'.join([meta_path, n])
keys = self.zke.get_children(keys_path)
with lock:
if n not in st['existed']:
st['existed'][n] = {}
for k in keys:
path = '/'.join([keys_path, k])
yield (n, k, path)
def _set_record(args):
try:
n, k, path = args
try:
with cachepool.make_wrapper(zke_pool)() as zke:
val, _ = zke.get(path)
except NoNodeError:
# the node may be deleted after get children
logger.warn('{p} has been deleted'.format(p=path))
return
# k='record_dir/meta/server/<server_id>
# remove record_dir
path = path[len(record_dir):]
self.storage.apply_record(path, val[-1])
with lock:
st['existed'][n][k] = 1
except Exception as e:
logger.exception(
repr(e) + ' while set record {a}'.format(a=args))
with lock:
st['failed'].append(args)
jobq.run(_iter_key(), [(_set_record, 20)])
if len(st['failed']) > 0:
logger.error('failed to set record {f}'.format(f=st['failed']))
raise SlaveError('failed to set record')
while not zke_pool.queue.empty():
ele = zke_pool.queue.get()
zkutil.close_zk(ele)
self.storage.delete_absent_record(st['existed'])
self.storage.set_journal_id_set(self.zk_journal_id_set)
def __init__(self, zk, txid=None, timeout=DEFAULT_TIMEOUT, lock_timeout=None):
# Save the original arg for self.run()
self._zk = zk
self.zke, self.owning_zk = zkutil.kazoo_client_ext(zk)
self.timeout = timeout
self.lock_timeout = lock_timeout
self.txid = txid
self.expire_at = None
self.tx_alive_lock = None
# User locked and required keys
self.got_keys = {}
# Keys user need to commit
self.modifications = {}
self.state_lock = threading.RLock()
self.connected = True
self.tx_status = None
self.zkstorage = ZKStorage(self.zke)
self.zkstorage.max_value_history = 2
self.zke.add_listener(self._on_conn_change)
def setUp(self):
super(TestZKStorage, self).setUp()
self.zk.create('tx', 'tx')
self.zk.create('lock', '{}')
self.zk.create('tx/journal', '{}')
self.zk.create('record', '{}')
zke, _ = zkutil.kazoo_client_ext(self.zk)
self.zs = zktx.ZKStorage(zke)
def setUp(self):
super(TestZKStorage, self).setUp()
self.zk.create('tx', 'tx')
self.zk.create('lock', '{}')
self.zk.create('tx/journal', '{}')
self.zk.create('record', '{}')
zke, _ = zkutil.kazoo_client_ext(self.zk)
self.zs = zktx.ZKStorage(zke)
def setUp(self):
utdocker.create_network()
utdocker.start_container(
zk_name,
zk_tag,
env={
"ZOO_MY_ID": 1,
"ZOO_SERVERS": "server.1=0.0.0.0:2888:3888",
},
port_bindings={2181: 21811}
)
self.zk = KazooClient('127.0.0.1:21811')
self.zk.start()
self.zkauthed, _ = zkutil.kazoo_client_ext(
{'hosts': '127.0.0.1:21811', 'auth': ('digest', 'xp', '123'),
'acl': (('xp', '123', 'cdrwa'), ('foo', 'bar', 'rw'))})
dd('start zk-test in docker')
def setUp(self):
utdocker.create_network()
utdocker.start_container(zk_name,
zk_tag,
env={
"ZOO_MY_ID": 1,
"ZOO_SERVERS":
"server.1=0.0.0.0:2888:3888",
},
port_bindings={2181: 21811})
self.zk = KazooClient('127.0.0.1:21811')
self.zk.start()
self.zkauthed, _ = zkutil.kazoo_client_ext({
'hosts':
'127.0.0.1:21811',
'auth': ('digest', 'xp', '123'),
'acl': (('xp', '123', 'cdrwa'), ('foo', 'bar', 'rw'))
})
dd('start zk-test in docker')