def test_same_ident_same_job(self):
ident = self.get_random_ident()
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'foo',
'sleep_time': 10,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
self.assertEqual('foo', resp['result'])
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'bar',
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
ws = self._create_client()
ws.send(utfjson.dump(job_desc))
resp = utfjson.load(ws.recv())
ws.close()
# not bar, because the ident is same as the first job,
# if job exists, it will not create a new one
self.assertEqual('foo', resp['result'])
self.assertEqual('foo', resp['echo'])
def test_same_ident_different_job(self):
ident = self.get_random_ident()
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'foo',
'sleep_time': 0.1,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
self.assertEqual('foo', resp['result'])
time.sleep(0.2)
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'bar',
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
ws2 = self._create_client()
ws2.send(utfjson.dump(job_desc))
resp = utfjson.load(ws2.recv())
ws2.close()
# old job with the same ident has exit, it will create a new one
self.assertEqual('bar', resp['result'])
self.assertEqual('bar', resp['echo'])
def test_check_system_load(self):
job_desc = {
'func': 'test_job_normal.run',
'ident': self.get_random_ident(),
'check_load': {
'mem_low_threshold': 100 * 1024**3,
'cpu_low_threshold': 0,
},
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = self.ws.recv()
resp = utfjson.load(resp)
self.assertEqual('SystemOverloadError', resp['err'])
job_desc = {
'func': 'test_job_normal.run',
'ident': self.get_random_ident(),
'check_load': {
'cpu_low_threshold': 100.1,
'mem_low_threshold': 0,
},
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
ws2 = self._create_client()
ws2.send(utfjson.dump(job_desc))
resp = ws2.recv()
resp = utfjson.load(resp)
ws2.close()
self.assertEqual('SystemOverloadError', resp['err'])
def test_check_system_load(self):
job_desc = {
'func': 'test_job_normal.run',
'ident': self.get_random_ident(),
'check_load': {
'mem_low_threshold': 100 * 1024 ** 3,
'cpu_low_threshold': 0,
},
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = self.ws.recv()
resp = utfjson.load(resp)
self.assertEqual('SystemOverloadError', resp['err'])
job_desc = {
'func': 'test_job_normal.run',
'ident': self.get_random_ident(),
'check_load': {
'cpu_low_threshold': 100.1,
'mem_low_threshold': 0,
},
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
ws2 = self._create_client()
ws2.send(utfjson.dump(job_desc))
resp = ws2.recv()
resp = utfjson.load(resp)
ws2.close()
self.assertEqual('SystemOverloadError', resp['err'])
def test_same_ident_different_job(self):
ident = self.get_random_ident()
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'foo',
'sleep_time': 0.1,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
self.assertEqual('foo', resp['result'])
time.sleep(0.2)
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'bar',
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
ws2 = self._create_client()
ws2.send(utfjson.dump(job_desc))
resp = utfjson.load(ws2.recv())
ws2.close()
# old job with the same ident has exit, it will create a new one
self.assertEqual('bar', resp['result'])
self.assertEqual('bar', resp['echo'])
def test_same_ident_same_job(self):
ident = self.get_random_ident()
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'foo',
'sleep_time': 10,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
self.assertEqual('foo', resp['result'])
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'bar',
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
ws = self._create_client()
ws.send(utfjson.dump(job_desc))
resp = utfjson.load(ws.recv())
ws.close()
# not bar, because the ident is same as the first job,
# if job exists, it will not create a new one
self.assertEqual('foo', resp['result'])
self.assertEqual('foo', resp['echo'])
def test_load_backslash_x_encoded(self):
s = '"\\x61"'
self.assertEqual('a', utfjson.load(s))
s = '"\\X61"'
self.assertEqual('a', utfjson.load(s))
s = '"\\xe6\\x88\\x91"'
self.assertEqual('我', utfjson.load(s))
self.assertRaises(utfjson.JSONDecodeError, utfjson.load, '"\\"')
self.assertRaises(utfjson.JSONDecodeError, utfjson.load, '"\\x"')
self.assertRaises(utfjson.JSONDecodeError, utfjson.load, '"\\x6"')
def test_load_backslash_x_encoded(self):
s = '"\\x61"'
self.assertEqual('a', utfjson.load(s))
s = '"\\X61"'
self.assertEqual('a', utfjson.load(s))
s = '"\\xe6\\x88\\x91"'
self.assertEqual('我', utfjson.load(s))
self.assertRaises(utfjson.JSONDecodeError, utfjson.load, '"\\"')
self.assertRaises(utfjson.JSONDecodeError, utfjson.load, '"\\x"')
self.assertRaises(utfjson.JSONDecodeError, utfjson.load, '"\\x6"')
def test_json(self):
region = Region({
'range': ['a', 'z'],
'levels': [
[['a', 'b', tbid1], ['b', 'c', tbid2]]
]})
rst = utfjson.dump(region)
expected = ('{"range": ["a", "z"], "levels": '
'[[["a", "b", "' + tbid1 + '"], ["b", "c", "' + tbid2 + '"]]], "idc": ""}')
self.assertEqual(utfjson.load(expected), region)
self.assertEqual(region, Region(utfjson.load(rst)))
def test_empty_commit_force(self):
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = {'foo': 'foo'}
t1.commit(force=True)
rst, ver = self.zk.get('tx/journal_id_set')
self.assertEqual({COMMITTED: [[0, 1]],
PURGED: [],
}, utfjson.load(rst))
rst, ver = self.zk.get('tx/journal/journal_id0000000000')
self.assertEqual({}, utfjson.load(rst))
def test_empty_commit_force(self):
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = {'foo': 'foo'}
t1.commit(force=True)
rst, ver = self.zk.get('tx/txidset')
self.assertEqual({COMMITTED: [[1, 2]],
PURGED: [],
}, utfjson.load(rst))
rst, ver = self.zk.get('tx/journal/0000000001')
self.assertEqual({}, utfjson.load(rst))
def try_release(self):
logger.debug(
'try to release if I am locker holder: {s}'.format(s=str(self)))
try:
holder, zstat = self.zkclient.get(self.lock_path)
holder = utfjson.load(holder)
self.lock_holder = (holder, zstat.version)
logger.debug('got lock holder: {s}'.format(s=str(self)))
if self.cmp_identifier(holder, self.identifier):
self.zkclient.remove_listener(self.on_connection_change)
try:
self.zkclient.delete(self.lock_path, version=zstat.version)
except NoNodeError as e:
logger.info(repr(e) + ' while delete lock: ' + str(self))
self.lock_holder = None
return True, holder, zstat.version
else:
return False, holder, zstat.version
except NoNodeError as e:
logger.info(
repr(e) + ' while try_release: {s}'.format(s=str(self)))
return True, self.identifier, -1
def _acquire_by_get(self):
logger.debug('to get: {s}'.format(s=str(self)))
try:
with self.mutex:
holder, zstat = self.zkclient.get(self.lock_path,
watch=self.on_node_change)
holder = utfjson.load(holder)
self.lock_holder = (holder, zstat.version)
logger.debug('got lock holder: {s}'.format(s=str(self)))
if self.cmp_identifier(holder, self.identifier):
logger.info('ACQUIRED: {s}'.format(s=str(self)))
return
logger.debug('other holds: {s}'.format(s=str(self)))
self.maybe_available.clear()
except NoNodeError as e:
# create failed but when getting it, it has been deleted
logger.info(repr(e) + ' while get lock: {s}'.format(s=str(self)))
with self.mutex:
self.lock_holder = None
self.maybe_available.set()
def update_conf(event):
logger.info('update conf triggered at: %f' % time.time())
context = global_value['context']
zk_path = '%s/arch_conf' % context['zk_prefix']
while True:
try:
zk_client = context['zk_client']
resp = zk_client.get(zk_path, watch=update_conf)
break
except Exception as e:
logger.exception('failed to get from zk: ' + repr(e))
time.sleep(5)
context['arch_conf'] = {
'version': resp[1].version,
'value': utfjson.load(resp[0]),
}
logger.info('arch conf in zk changed at: %f, current verrsion: %d' %
(time.time(), resp[1].version))
cgroup_manager.build_all_subsystem_cgroup_arch(context)
def handle(cls, response):
body = response.data
e = {}
e['status'] = response.status
e['headers'] = response.headers
e['response'] = body
try:
r = utfjson.load(body)
except ValueError:
r = {"message": "response body is not json", "cause": str(body)}
ecode = r.get('errorCode')
default_exc = EtcdException
if response.status == 404:
ecode = 100
elif response.status == 401:
ecode = 110
elif response.status >= 500:
default_exc = EtcdResponseError
exc = cls.error_exceptions.get(ecode, default_exc)
if ecode in cls.error_exceptions:
msg = "{msg} : {cause}".format(msg=r.get('message'),
cause=r.get('cause'))
else:
msg = "Unable to decode server response"
e['message'] = msg
raise exc(e)
def test_json(self):
region = Region({
'range': ['a', 'z'],
'levels':
[[['a', 'b', BlockDesc()],
[
'b', 'c',
BlockDesc(
size=2,
block_id=BlockID(
'd1g0006300000001230101idc000c62d8736c72800020000000001'
))
]]]
})
rst = utfjson.dump(region)
expected = (
'{"range": ["a", "z"], "levels": [[["a", "b", '
'{"is_del": 0, "range": null, "block_id": null, "size": 0}], '
'["b", "c", {"is_del": 0, "range": null, '
'"block_id": "d1g0006300000001230101idc000c62d8736c72800020000000001", "size": 2}]]], "idc": ""}'
)
self.assertEqual(expected, rst)
loaded = Region(utfjson.load(rst))
self.assertEqual(region, loaded)
def init_arch_conf(context):
while True:
try:
if context['zk_client'] is None:
context['zk_client'] = get_zk_client(context)
context['zk_client'].add_listener(on_lost)
zk_path = '%s/arch_conf' % context['zk_prefix']
resp = context['zk_client'].get(zk_path, watch=update_conf)
context['arch_conf'] = {
'version': resp[1].version,
'value': utfjson.load(resp[0]),
}
return
except Exception as e:
logger.warn('failed to get arch conf from zk: %s' % repr(e))
try:
context['zk_client'].stop()
except Exception as e:
logger.info('failed to stop zk client: ' + repr(e))
context['zk_client'] = None
time.sleep(10)
def test_config(self):
old = (config.zk_acl, config.zk_auth, config.zk_node_id)
config.zk_acl = (('foo', 'bar', 'cd'),
('xp', '123', 'cdrwa'))
config.zk_auth = ('digest', 'xp', '123')
config.zk_node_id = 'abc'
l = zkutil.ZKLock('foo_name', on_lost=lambda: True)
dd(l.zkconf.acl())
def _check_ac(ac):
self.assertEqual('digest', ac.id.scheme)
self.assertEqual('foo', ac.id.id.split(':')[0])
self.assertEqual(set(['CREATE', 'DELETE']), set(ac.acl_list))
_check_ac(l.zkconf.kazoo_digest_acl()[0])
with l:
# should have created lock node
data, zstate = self.zk.get(l.lock_path)
data = utfjson.load(data)['id']
dd(data)
self.assertEqual('abc', data.split('-')[0])
acls, zstate = self.zk.get_acls(l.lock_path)
dd(acls)
_check_ac(acls[0])
(config.zk_acl, config.zk_auth, config.zk_node_id) = old
def test_client_close(self):
ident = self.get_random_ident()
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'foo',
'time_sleep': 10,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
self.ws.close()
self.ws = self._create_client()
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'bar',
'time_sleep': 10,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = self.ws.recv()
resp = utfjson.load(resp)
self.assertEqual('foo', resp['result'])
def test_invalid_jobdesc(self):
cases = (
('foo', 'not json'),
(utfjson.dump('foo'), 'not dict'),
(utfjson.dump({}), 'no func'),
(utfjson.dump({'func': 'foo'}), 'no ident'),
(utfjson.dump({'ident': 'bar'}), 'no func'),
(utfjson.dump({
'ident': 'bar',
'func': {}
}), 'invalid func'),
(utfjson.dump({
'ident': 44,
'func': 'foo'
}), 'invalid ident'),
(utfjson.dump({
'ident': 'foo',
'func': 'foo',
'jobs_dir': {}
}), 'invalid jobs_dir'),
)
for msg, desc in cases:
ws = self._create_client()
ws.send(msg)
resp = utfjson.load(ws.recv())
self.assertIn('err', resp, desc)
ws.close()
def test_invalid_check_load_args(self):
cases = (
{'check_load': {'mem_low_threshold': 'foo',
'cpu_low_threshold': 0}},
{'check_load': {'cpu_low_threshold': None,
'mem_low_threshold': 0}},
{'check_load': {'max_client_number': {},
'cpu_low_threshold': 0,
'mem_low_threshold': 0}},
)
job_desc = {
'func': 'test_job_normal.run',
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
for case in cases:
case.update(job_desc)
case['ident'] = self.get_random_ident()
ws = self._create_client()
ws.send(utfjson.dump(case))
resp = utfjson.load(ws.recv())
ws.close()
self.assertEqual('InvalidMessageError', resp['err'])
def test_json(self):
blk = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'ts_range': ["1235", "456"],
'ref_num':
1,
'is_del':
0,
'mtime':
1,
})
rst = utfjson.dump(blk)
expected = (
'{"block_id": "d0g0006400000001230000idc000c62d8736c72800020000000001", "is_del": 0, "ref_num": 1, "range": ["0a", "0b"], "mtime": 1, "ts_range": ["1235", "456"], "size": 1000}'
)
self.assertEqual(expected, rst)
loaded = BlockDesc(utfjson.load(rst))
self.assertEqual(blk, loaded)
def test_client_close(self):
ident = self.get_random_ident()
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'foo',
'time_sleep': 10,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
self.ws.close()
self.ws = self._create_client()
job_desc = {
'func': 'test_job_echo.run',
'ident': ident,
'echo': 'bar',
'time_sleep': 10,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = self.ws.recv()
resp = utfjson.load(resp)
self.assertEqual('foo', resp['result'])
def recv_msg(self):
v = self.rcl.lpop(self.recv_list_name)
if v is None:
return None
return utfjson.load(v)
def recv_msg(self):
v = self.rcl.lpop(self.recv_list_name)
if v is None:
return None
return utfjson.load(v)
def test_deepcopy_value(self):
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = {'foo': 'foo'}
t1.set(foo)
t1.commit()
with ZKTransaction(zkhost) as t1:
# When retrieving a dict, it should be deep-copied.
# All zktx always though modified == original
foo = t1.lock_get('foo')
foo.v['foo'] = 'bar'
t1.set(foo)
t1.commit()
rst, ver = self.zk.get('record/foo')
self.assertEqual([{
'foo': 'foo'
}, {
'foo': 'bar'
}],
utfjson.load(rst)[-2:])
def handle(cls, response):
body = response.data
e = {}
e['status'] = response.status
e['headers'] = response.headers
e['response'] = body
try:
r = utfjson.load(body)
except ValueError:
r = {"message": "response body is not json", "cause": str(body)}
ecode = r.get('errorCode')
default_exc = EtcdException
if response.status == 404:
ecode = 100
elif response.status == 401:
ecode = 110
elif response.status >= 500:
default_exc = EtcdResponseError
exc = cls.error_exceptions.get(ecode, default_exc)
if ecode in cls.error_exceptions:
msg = "{msg} : {cause}".format(msg=r.get('message'),
cause=r.get('cause'))
else:
msg = "Unable to decode server response"
e['message'] = msg
raise exc(e)
def update_conf(event):
logger.info('update conf triggered at: %f' % time.time())
context = global_value['context']
zk_path = '%s/arch_conf' % context['zk_prefix']
while True:
try:
zk_client = context['zk_client']
resp = zk_client.get(zk_path, watch=update_conf)
break
except Exception as e:
logger.exception('failed to get from zk: ' + repr(e))
time.sleep(5)
context['arch_conf'] = {
'version': resp[1].version,
'value': utfjson.load(resp[0]),
}
logger.info('arch conf in zk changed at: %f, current verrsion: %d' %
(time.time(), resp[1].version))
cgroup_manager.build_all_subsystem_cgroup_arch(context)
def hget(self, hashname, hashkey):
hashname = self._get_path(hashname)
val = self.cli.hget(hashname, hashkey)
if self.load is not None:
return self.load(val)
return utfjson.load(val)
def get(self, key):
key = self._get_path(key)
val = self.cli.get(key)
if self.load is not None:
return self.load(val)
return utfjson.load(val)
def _wait_for_result(self, ws):
# wait for test_job_echo.run to fillin resp['result']
for _ in range(3):
resp = utfjson.load(ws.recv())
if 'result' in resp:
break
time.sleep(0.1)
return resp
def test_load_decode(self):
self.assertEqual('我', utfjson.load('"我"'))
self.assertEqual(u'我', utfjson.load('"我"', encoding='utf-8'))
self.assertEqual(unicode, type(utfjson.load('"我"', encoding='utf-8')))
self.assertEqual({'a': u"我".encode('utf-8')}, utfjson.load('{"a": "\\u6211"}'))
self.assertEqual({'a': u"我"}, utfjson.load('{"a": "我"}', encoding='utf-8'))
self.assertEqual({'a': "我"}, utfjson.load('{"a": "我"}'))
self.assertEqual({'a': "我"}, utfjson.load('{"a": "我"}'))
self.assertEqual(["我"], utfjson.load('["我"]'))
def test_with_statement(self):
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = 1
t1.set(foo)
t1.commit()
rst, ver = self.zk.get('record/foo')
self.assertEqual([[-1, None], [1, 1]], utfjson.load(rst))
rst, ver = self.zk.get('tx/txidset')
self.assertEqual({COMMITTED: [[1, 2]],
PURGED: [],
}, utfjson.load(rst))
def recv_last_msg(self):
last = None
while True:
v = self.rcl.lpop(self.recv_list_name)
if v is None:
return utfjson.load(last)
last = v
def recv_last_msg(self):
last = None
while True:
v = self.rcl.lpop(self.recv_list_name)
if v is None:
return utfjson.load(last)
last = v
def test_with_statement(self):
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = 1
t1.set(foo)
t1.commit()
rst, ver = self.zk.get('record/foo')
self.assertEqual([None, 1], utfjson.load(rst))
rst, ver = self.zk.get('tx/journal_id_set')
self.assertEqual({COMMITTED: [[0, 1]],
PURGED: [],
}, utfjson.load(rst))
def load_progress():
progress_file = cnf['PROGRESS_FILE']
if os.path.isfile(progress_file):
progress = utfjson.load(fsutil.read_file(progress_file))
else:
progress = {'marker': '', 'total_n': 0, 'total_size': 0.}
return progress
def test_empty_commit(self):
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = {'foo': 'foo'}
t1.commit()
rst, ver = self.zk.get('tx/journal_id_set')
self.assertEqual({}, utfjson.load(rst))
def brecv_msg(self, timeout=0):
v = self.rcl.blpop(self.recv_list_name, timeout=timeout)
if v is None or len(v) != 2:
return None
# v is a tuple, (key, value)
_, value = v
return utfjson.load(value)
def brecv_msg(self, timeout=0):
v = self.rcl.blpop(self.recv_list_name, timeout=timeout)
if v is None or len(v) != 2:
return None
# v is a tuple, (key, value)
_, value = v
return utfjson.load(value)
def process_message(self, message_str):
message = utfjson.load(message_str)
cmd = message['cmd']
args = message.get('args')
if args is None:
args = {}
result = self.do_cmd(cmd, args)
self.send_json(result)
def test_empty_commit(self):
with ZKTransaction(zkhost) as t1:
foo = t1.lock_get('foo')
foo.v = {'foo': 'foo'}
t1.commit()
rst, ver = self.zk.get('tx/journal_id_set')
self.assertEqual({}, utfjson.load(rst))
def test_load_decode(self):
self.assertEqual('我', utfjson.load('"我"'))
self.assertEqual(u'我', utfjson.load('"我"', encoding='utf-8'))
self.assertEqual(unicode, type(utfjson.load('"我"', encoding='utf-8')))
self.assertEqual({'a': u"我".encode('utf-8')},
utfjson.load('{"a": "\\u6211"}'))
self.assertEqual({'a': u"我"},
utfjson.load('{"a": "我"}', encoding='utf-8'))
self.assertEqual({'a': "我"}, utfjson.load('{"a": "我"}'))
self.assertEqual({'a': "我"}, utfjson.load('{"a": "我"}'))
self.assertEqual(["我"], utfjson.load('["我"]'))
def process_message(self, message_str):
message = utfjson.load(message_str)
cmd = message['cmd']
args = message.get('args')
if args is None:
args = {}
result = self.do_cmd(cmd, args)
self.send_json(result)
def test_apply_jour(self):
cases = (
{
'meta/h1/k1': 1
},
{
'meta/h2/k2': {}
},
{
'meta/h3/k3': []
},
{
'meta/h4/k4': 'foo'
},
{
'meta/h5/k5': {
'xx': 'xxx'
}
},
{
'meta/h10/k10': 1,
'meta/h100/k100': 10
},
{
'meta/h11/k11': {},
'meta/h101/k101': {}
},
{
'meta/h12/k12': [],
'meta/h102/k102': []
},
{
'meta/h13/k13': 'foo',
'meta/h103/k103': 'bar'
},
{
'meta/h14/k14': {
'xx': 'xxx'
},
'meta/h104/k104': {
'yy': 'yyy'
}
},
)
for jour in cases:
self.storage.apply_jour(jour)
for k, v in jour.items():
k_parts = k.split('/', 2)
actual = self.redis_cli.hget(k_parts[1], k_parts[2])
self.assertEqual(v, utfjson.load(actual))
def test_module_not_exists(self):
job_desc = {
'func': 'foo.bar',
'ident': self.get_random_ident(),
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
self.assertIn('err', resp)
def test_module_not_exists(self):
job_desc = {
'func': 'foo.bar',
'ident': self.get_random_ident(),
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
self.assertIn('err', resp)
def test_json(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
rst = utfjson.dump(g)
expected = ('{"config": {"in_idc": [4, 2], "ec_policy": "lrc", "cross_idc": [2, 1], '
'"data_replica": 3}, "blocks": {}, "idcs": ["a", "b", "c"], '
'"block_group_id": "g000640000000123"}')
self.assertEqual(expected, rst)
loaded = BlockGroup(utfjson.load(rst))
self.assertEqual(g, loaded)
def test_single_record(self):
tx = ZKTransaction(zkhost)
tx.begin()
foo = tx.lock_get('foo')
self.assertEqual('foo', foo.k)
self.assertIsNone(foo.v)
foo.v = 1
tx.set(foo)
tx.commit()
rst, ver = self.zk.get('record/foo')
self.assertEqual([None, 1], utfjson.load(rst))
rst, ver = self.zk.get('tx/journal_id_set')
self.assertEqual({COMMITTED: [[0, 1]],
PURGED: [],
}, utfjson.load(rst))
def test_invalid_cpu_sample_interval(self):
job_desc = {
'func': 'test_job_normal.run',
'ident': self.get_random_ident(),
'cpu_sample_interval': 'foo',
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
self.assertEqual('InvalidMessageError', resp['err'])
def test_func_not_exists(self):
ident = self.get_random_ident()
job_desc = {
'func': 'test_job_echo.func_not_exists',
'ident': ident,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = self.ws.recv()
resp = utfjson.load(resp)
self.assertEqual('LoadingError', resp['err'])
def test_normal_job(self):
job_desc = {
'func': 'test_job_normal.run',
'ident': self.get_random_ident(),
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
dd(resp)
self.assertEqual('foo', resp['result'], 'test get result')
def _to_dict(self, response):
try:
return utfjson.load(response.data)
except ValueError as e:
logger.error(repr(e) + ' while decode {data}'.format(
data=response.data))
raise EtcdIncompleteRead('failed to decode %s' % response.data)
except Exception as e:
logger.error(repr(e) + ' while decode {data}'.format(
data=response.data))
raise EtcdResponseError('failed to decode %s' % response.data)
def try_release_key(self, txid, key):
logger.info('releasing: {txid} {key}'.format(txid=txid, key=key))
keylock = self._make_key_lock(txid, key)
locked, txid, ver = keylock.try_acquire()
if locked:
keylock.release()
else:
keylock.close()
return locked, utfjson.load(txid), ver
def test_report_system_load(self):
job_desc = {
'func': 'test_job_normal.run',
'ident': self.get_random_ident(),
'report_system_load': True,
'jobs_dir': 'pykit/wsjobd/test/test_jobs',
}
self.ws.send(utfjson.dump(job_desc))
resp = utfjson.load(self.ws.recv())
self.assertIn('mem_available', resp['system_load'])
self.assertIn('cpu_idle_percent', resp['system_load'])
self.assertIn('client_number', resp['system_load'])
def test_journal(self):
self.zs.journal.create(1, {'a': 'b'})
rst, ver = self.zs.journal.get(1)
self.assertEqual({'a': 'b'}, rst)
rst, ver = self.zk.get('tx/journal/journal_id0000000001')
self.assertEqual({'a': 'b'}, utfjson.load(rst))
self.assertRaises(BadVersionError, self.zs.journal.delete, 1, version=100)
self.zs.journal.delete(1, version=0)
self.assertRaises(NoNodeError, self.zs.journal.get, 1)
def leader(self):
res = self.api_execute(self._stats_path + '/self', self._MGET)
self_st = utfjson.load(res.data)
leader_id = self_st.get('leaderInfo', {}).get('leader')
if leader_id is None:
return None
mems = self.members
for mem in mems:
if mem['id'] != leader_id:
continue
return mem.copy()
def _to_keysresult(self, response):
try:
res = utfjson.load(response.data)
r = EtcdKeysResult(**res)
r.parse_response(response)
return r
except ValueError as e:
logger.error(repr(e) + ' while decode {data}'.format(
data=response.data))
raise EtcdIncompleteRead('failed to decode %s' % response.data)
except Exception as e:
logger.error(repr(e) + ' while decode {data}'.format(
data=response.data))
raise EtcdResponseError('failed to decode %s' % response.data)