def replay_patched_node(looper, replaying_node, node_recorder, cr):
node_run_no = 0
looper.add(replaying_node)
cr.start_playing()
next_stop_at = time.perf_counter() + (cr.start_times[node_run_no][1] -
cr.start_times[node_run_no][0])
while cr.is_playing:
vals = cr.get_next()
if next_stop_at is not None and time.perf_counter() >= next_stop_at:
node_run_no += 1
if node_run_no < len(cr.start_times):
# The node stopped here
sleep_for = cr.start_times[node_run_no][0] - cr.start_times[
node_run_no - 1][1]
replaying_node.stop()
looper.removeProdable(replaying_node)
# Create new node since node is destroyed on stop
replaying_node = replaying_node.__class__(
replaying_node.name,
config_helper=replaying_node.config_helper,
ha=replaying_node.nodestack.ha,
cliha=replaying_node.clientstack.ha)
patch_replaying_node(replaying_node, node_recorder,
cr.start_times)
print(
'sleeping for {} to simulate node stop'.format(sleep_for))
time.sleep(sleep_for)
after = cr.start_times[node_run_no][1] - cr.start_times[
node_run_no][0]
next_stop_at = time.perf_counter() + after
print('Next stop after {}'.format(after))
looper.add(replaying_node)
else:
next_stop_at = None
if not vals:
continue
n_msgs, c_msgs = vals
if n_msgs:
for inc in n_msgs:
if Recorder.is_incoming(inc):
msg, frm = to_bytes(inc[1]), to_bytes(inc[2])
replaying_node.nodestack._verifyAndAppend(msg, frm)
if Recorder.is_disconn(inc):
disconnecteds = inc[1:]
replaying_node.nodestack._connsChanged(
set(), disconnecteds)
if c_msgs:
incomings = Recorder.filter_incoming(c_msgs)
for inc in incomings:
msg, frm = to_bytes(inc[0]), to_bytes(inc[1])
replaying_node.clientstack._verifyAndAppend(msg, frm)
looper.run(replaying_node.prod())
return replaying_node
def patch_sent_prepreapres(replaying_node, node_recorder):
sent_pps = {}
def add_preprepare(msg):
inst_id, v, p = msg[f.INST_ID.nm], msg[f.VIEW_NO.nm], msg[
f.PP_SEQ_NO.nm]
if inst_id not in sent_pps:
sent_pps[inst_id] = {}
sent_pps[inst_id][v, p] = [
msg[f.PP_TIME.nm],
[l for l in msg[f.REQ_IDR.nm]],
msg[f.DISCARDED.nm],
]
msg_count = 0
min_msg_time = sys.maxsize
max_msg_time = -1
for k, v in node_recorder.store.iterator(include_value=True):
max_msg_time = max(max_msg_time, int(k))
min_msg_time = min(min_msg_time, int(k))
parsed = Recorder.get_parsed(v.decode())
msg_count += len(parsed)
outgoings = Recorder.filter_outgoing(parsed)
if not outgoings:
continue
for out in outgoings:
try:
msg = json.loads(out[0])
if isinstance(msg, dict) and OP_FIELD_NAME in msg:
op_name = msg[OP_FIELD_NAME]
if op_name == PREPREPARE:
add_preprepare(msg)
elif op_name == BATCH:
for m in msg['messages']:
try:
m = json.loads(m)
if m[OP_FIELD_NAME] == PREPREPARE:
add_preprepare(m)
except json.JSONDecodeError:
continue
else:
continue
except json.JSONDecodeError:
continue
for r in replaying_node.replicas.values():
r.sent_pps = sent_pps.pop(r.instId, {})
replaying_node.sent_pps = sent_pps
replaying_node.replay_msg_count = msg_count
run_time = max_msg_time - min_msg_time
run_time = int(run_time / Recorder.TIME_FACTOR)
print("Aprox run time: {}".format(str(
datetime.timedelta(seconds=run_time))))
def patch_sent_prepreapres(replaying_node, node_recorder):
sent_pps = {}
def add_preprepare(msg):
inst_id, v, p = msg[f.INST_ID.nm], msg[f.VIEW_NO.nm], msg[
f.PP_SEQ_NO.nm]
if inst_id not in sent_pps:
sent_pps[inst_id] = {}
sent_pps[inst_id][v, p] = [msg[f.PP_TIME.nm],
[l for l in
msg[f.REQ_IDR.nm]],
msg[f.DISCARDED.nm],
]
msg_count = 0
min_msg_time = sys.maxsize
max_msg_time = -1
for k, v in node_recorder.store.iterator(include_value=True):
max_msg_time = max(max_msg_time, int(k))
min_msg_time = min(min_msg_time, int(k))
parsed = Recorder.get_parsed(v.decode())
msg_count += len(parsed)
outgoings = Recorder.filter_outgoing(parsed)
if not outgoings:
continue
for out in outgoings:
try:
msg = json.loads(out[0])
if isinstance(msg, dict) and OP_FIELD_NAME in msg:
op_name = msg[OP_FIELD_NAME]
if op_name == PREPREPARE:
add_preprepare(msg)
elif op_name == BATCH:
for m in msg['messages']:
try:
m = json.loads(m)
if m[OP_FIELD_NAME] == PREPREPARE:
add_preprepare(m)
except json.JSONDecodeError:
continue
else:
continue
except json.JSONDecodeError:
continue
for r in replaying_node.replicas.values():
r.sent_pps = sent_pps.pop(r.instId, {})
replaying_node.sent_pps = sent_pps
replaying_node.replay_msg_count = msg_count
run_time = max_msg_time - min_msg_time
run_time = int(run_time / Recorder.TIME_FACTOR)
print("Aprox run time: {}".format(str(datetime.timedelta(seconds=run_time))))
def get_recorders_from_node_data_dir(node_data_dir, node_name) -> Tuple[Recorder, Recorder]:
rec_path = os.path.join(node_data_dir, node_name, 'recorder')
client_stack_name = node_name + CLIENT_STACK_SUFFIX
client_rec_kv_store = initKeyValueStorageIntKeys(KeyValueStorageType.Rocksdb,
rec_path, client_stack_name)
node_rec_kv_store = initKeyValueStorageIntKeys(
KeyValueStorageType.Rocksdb, rec_path, node_name)
return Recorder(node_rec_kv_store, skip_metadata_write=True), \
Recorder(client_rec_kv_store, skip_metadata_write=True)
def get_parsed(msg, only_incoming=None, only_outgoing=None):
# To conform to the Recorder interface
assert only_incoming is None
assert only_outgoing is None
n_msgs, c_msgs = msg.split(CombinedRecorder.separator)
n_msgs = n_msgs.lstrip(CombinedRecorder.n_prefix)
c_msgs = c_msgs.lstrip(CombinedRecorder.c_prefix)
return [Recorder.get_parsed(n_msgs) if n_msgs else [],
Recorder.get_parsed(c_msgs) if c_msgs else []]
def __init__(self, *args, **kwargs):
parent_dir, _ = os.path.split(args[0]['basedirpath'])
name = args[0]['name']
from stp_core.network.keep_in_touch import KITNetworkInterface
if isinstance(self, KITNetworkInterface):
db_path = os.path.join(parent_dir, 'data', name, 'recorder')
else:
db_path = os.path.join(parent_dir, 'data', name[:-1], 'recorder')
os.makedirs(db_path, exist_ok=True)
db = KeyValueStorageRocksdbIntKeys(db_path, name)
self.recorder = Recorder(db)
super().__init__(*args, **kwargs)
class SimpleZStackWithRecorder(SimpleZStack):
# Used during recording
def __init__(self, *args, **kwargs):
parent_dir, _ = os.path.split(args[0]['basedirpath'])
name = args[0]['name']
from stp_core.network.keep_in_touch import KITNetworkInterface
if isinstance(self, KITNetworkInterface):
db_path = os.path.join(parent_dir, 'data', name, 'recorder')
else:
db_path = os.path.join(parent_dir, 'data', name[:-1], 'recorder')
os.makedirs(db_path, exist_ok=True)
db = KeyValueStorageRocksdbIntKeys(db_path, name)
self.recorder = Recorder(db)
super().__init__(*args, **kwargs)
def _verifyAndAppend(self, msg, ident):
if super()._verifyAndAppend(msg, ident):
logger.trace('{} recording incoming {} from {}'.format(self, msg, ident))
self.recorder.add_incoming(msg, ident)
def transmit(self, msg, uid, timeout=None, serialized=False):
status, err = super().transmit(msg, uid, timeout=timeout, serialized=serialized)
if status:
self.recorder.add_outgoing(msg, uid)
return status, err
def _connsChanged(self, ins: Set[str], outs: Set[str]) -> None:
from plenum.common.stacks import KITZStack
if isinstance(self, KITZStack) and outs:
self.recorder.add_disconnecteds(*outs)
super()._connsChanged(ins, outs)
def stop(self):
self.recorder.stop()
super().stop()
def test_recorder_get_next_incoming_only(recorder):
incoming_count = 100
incoming = [(randomString(100), randomString(6))
for _ in range(incoming_count)]
while incoming:
recorder.add_incoming(*incoming.pop())
time.sleep(random.choice([0, 1]) + random.random())
recorded_incomings = OrderedDict()
keys = []
for k, v in recorder.store.iterator(include_value=True):
v = Recorder.get_parsed(v)
keys.append(int(k))
recorded_incomings[int(k)] = v
assert len(recorded_incomings) == incoming_count
assert sorted(keys) == keys
max_time_to_run = incoming_count * 2 + 10
recorder.start_playing()
start = time.perf_counter()
while recorder.is_playing and (time.perf_counter() <
start + max_time_to_run):
vals = recorder.get_next()
if vals:
check = recorded_incomings.popitem(last=False)[1]
assert check == vals
else:
time.sleep(0.01)
assert len(recorded_incomings) == 0
assert not recorder.is_playing
def test_recorder_get_next_incoming_only(recorder):
incoming_count = 100
incoming = [(randomString(100), randomString(6)) for _ in
range(incoming_count)]
while incoming:
recorder.add_incoming(*incoming.pop())
time.sleep(random.choice([0, 1]) + random.random())
recorded_incomings = OrderedDict()
keys = []
for k, v in recorder.store.iterator(include_value=True):
v = Recorder.get_parsed(v)
keys.append(int(k))
recorded_incomings[int(k)] = v
assert len(recorded_incomings) == incoming_count
assert sorted(keys) == keys
max_time_to_run = incoming_count * 2 + 10
recorder.start_playing()
start = time.perf_counter()
while recorder.is_playing and (time.perf_counter() < start + max_time_to_run):
vals = recorder.get_next()
if vals:
check = recorded_incomings.popitem(last=False)[1]
assert check == vals
else:
time.sleep(0.01)
assert len(recorded_incomings) == 0
assert not recorder.is_playing
def test_recorder_get_next(recorder):
incoming_count = 100
outgoing_count = 50
incoming = [(randomString(100), randomString(6))
for _ in range(incoming_count)]
outgoing = [(randomString(100), randomString(6))
for _ in range(outgoing_count)]
while incoming or outgoing:
if random.choice([0, 1]) and outgoing:
recorder.add_outgoing(*outgoing.pop())
time.sleep(random.choice([0, 1]) + random.random())
elif incoming:
recorder.add_incoming(*incoming.pop())
time.sleep(random.choice([0, 1]) + random.random())
else:
continue
recorded_incomings = OrderedDict()
for k, v in recorder.store.iterator(include_value=True):
v = Recorder.get_parsed(v, only_incoming=True)
if v:
recorded_incomings[int(k)] = v
assert len(recorded_incomings) == incoming_count
max_time_to_run = incoming_count * 2 + 10
recorder.start_playing()
start = time.perf_counter()
while recorder.is_playing and (time.perf_counter() <
start + max_time_to_run):
vals = recorder.get_next()
if vals:
inc = Recorder.filter_incoming(vals)
if inc:
assert recorded_incomings.popitem(last=False)[1] == inc
else:
time.sleep(0.01)
assert len(recorded_incomings) == 0
assert not recorder.is_playing
def patch_sent_prepreapres(replaying_node, node_recorder):
sent_pps = {}
def add_preprepare(msg):
inst_id, v, p = msg[f.INST_ID.nm], msg[f.VIEW_NO.nm], msg[
f.PP_SEQ_NO.nm]
if inst_id not in sent_pps:
sent_pps[inst_id] = {}
sent_pps[inst_id][v, p] = [
msg[f.PP_TIME.nm],
[tuple(l) for l in msg[f.REQ_IDR.nm]],
msg[f.DISCARDED.nm],
]
for k, v in node_recorder.store.iterator(include_value=True):
parsed = Recorder.get_parsed(v.decode())
outgoings = Recorder.filter_outgoing(parsed)
if not outgoings:
continue
for out in outgoings:
try:
msg = json.loads(out[0])
if isinstance(msg, dict) and OP_FIELD_NAME in msg:
op_name = msg[OP_FIELD_NAME]
if op_name == PREPREPARE:
add_preprepare(msg)
elif op_name == BATCH:
for m in msg['messages']:
try:
m = json.loads(m)
if m[OP_FIELD_NAME] == PREPREPARE:
add_preprepare(m)
except json.JSONDecodeError:
continue
else:
continue
except json.JSONDecodeError:
continue
for r in replaying_node.replicas:
r.sent_pps = sent_pps.pop(r.instId, {})
replaying_node.sent_pps = sent_pps
def test_recorder_get_next(recorder):
incoming_count = 100
outgoing_count = 50
incoming = [(randomString(100), randomString(6)) for _ in range(incoming_count)]
outgoing = [(randomString(100), randomString(6)) for _ in range(outgoing_count)]
while incoming or outgoing:
if random.choice([0, 1]) and outgoing:
recorder.add_outgoing(*outgoing.pop())
time.sleep(random.choice([0, 1]) + random.random())
elif incoming:
recorder.add_incoming(*incoming.pop())
time.sleep(random.choice([0, 1]) + random.random())
else:
continue
recorded_incomings = OrderedDict()
for k, v in recorder.store.iterator(include_value=True):
v = Recorder.get_parsed(v, only_incoming=True)
if v:
recorded_incomings[int(k)] = v
assert len(recorded_incomings) == incoming_count
max_time_to_run = incoming_count * 2 + 10
recorder.start_playing()
start = time.perf_counter()
while recorder.is_playing and (time.perf_counter() < start + max_time_to_run):
vals = recorder.get_next()
if vals:
inc = Recorder.filter_incoming(vals)
if inc:
assert recorded_incomings.popitem(last=False)[1] == inc
else:
time.sleep(0.01)
assert len(recorded_incomings) == 0
assert not recorder.is_playing
def test_recorded_parsings(recorder):
incoming = [[randomString(10), randomString(6)] for i in range(3)]
outgoing = [[randomString(10), randomString(6)] for i in range(5)]
for m, f in incoming:
recorder.add_incoming(m, f)
time.sleep(0.01)
for m, f in outgoing:
recorder.add_outgoing(m, f)
time.sleep(0.01)
with pytest.raises(AssertionError):
recorder.get_parsed(incoming[0],
only_incoming=True,
only_outgoing=True)
combined = incoming + outgoing
def sublist(lst1, lst2):
ls1 = [element for element in lst1 if element in lst2]
ls2 = [element for element in lst2 if element in lst1]
return ls1 == ls2
for k, v in recorder.store.iterator(include_value=True):
p = Recorder.get_parsed(v)
assert sublist([i[1:] for i in p], combined)
p = Recorder.get_parsed(v, only_incoming=True)
if p:
assert sublist(p, incoming)
for i in p:
incoming.remove(i)
p = Recorder.get_parsed(v, only_outgoing=True)
if p:
assert sublist(p, outgoing)
for i in p:
outgoing.remove(i)
assert not incoming
assert not outgoing
def test_recorded_parsings(recorder):
incoming = [[randomString(10), randomString(6)] for i in
range(3)]
outgoing = [[randomString(10), randomString(6)] for i in
range(5)]
for m, f in incoming:
recorder.add_incoming(m, f)
time.sleep(0.01)
for m, f in outgoing:
recorder.add_outgoing(m, f)
time.sleep(0.01)
with pytest.raises(AssertionError):
recorder.get_parsed(incoming[0], only_incoming=True, only_outgoing=True)
combined = incoming + outgoing
def sublist(lst1, lst2):
ls1 = [element for element in lst1 if element in lst2]
ls2 = [element for element in lst2 if element in lst1]
return ls1 == ls2
for k, v in recorder.store.iterator(include_value=True):
p = Recorder.get_parsed(v)
assert sublist([i[1:] for i in p] , combined)
p = Recorder.get_parsed(v, only_incoming=True)
if p:
assert sublist(p, incoming)
for i in p:
incoming.remove(i)
p = Recorder.get_parsed(v, only_outgoing=True)
if p:
assert sublist(p, outgoing)
for i in p:
outgoing.remove(i)
assert not incoming
assert not outgoing
def __init__(self, kv_store: KeyValueStorageRocksdbIntKeys,
skip_metadata_write=True):
Recorder.__init__(self, kv_store, skip_metadata_write=skip_metadata_write)
self.recorders = []
self.start_times = []
def replay_patched_node(looper, replaying_node, node_recorder, cr):
node_run_no = 0
looper.add(replaying_node)
cr.start_playing()
next_stop_at = time.perf_counter() + (cr.start_times[node_run_no][1] -
cr.start_times[node_run_no][0])
progress_data = _create_progress_data(replaying_node.replay_msg_count)
#
# n_msg_count = 0
# start_time = time.perf_counter()
# next_progress_note = start_time + 5
while cr.is_playing:
_print_progress(progress_data)
vals = cr.get_next()
if next_stop_at is not None and time.perf_counter() >= next_stop_at:
node_run_no += 1
if node_run_no < len(cr.start_times):
# The node stopped here
sleep_for, after = _cal_run_times(node_run_no, cr.start_times)
replaying_node.stop()
looper.removeProdable(replaying_node)
# Create new node since node is destroyed on stop
replaying_node = replaying_node.__class__(replaying_node.name,
config_helper=replaying_node.config_helper,
ha=replaying_node.nodestack.ha,
cliha=replaying_node.clientstack.ha)
patch_replaying_node(replaying_node, node_recorder, cr.start_times)
print('Sleeping for {}s to simulate node stop'.format(sleep_for))
time.sleep(sleep_for)
if after is None:
next_stop_at = None
else:
next_stop_at = time.perf_counter() + after
print('Next stop after {}s'.format(after))
looper.add(replaying_node)
else:
next_stop_at = None
if not vals:
looper.run(replaying_node.prod())
continue
n_msgs, c_msgs = vals
progress_data = _update_progress_msg_count(progress_data, len(n_msgs))
if n_msgs:
for inc in n_msgs:
if Recorder.is_incoming(inc):
msg, frm = to_bytes(inc[1]), to_bytes(inc[2])
replaying_node.nodestack._verifyAndAppend(msg, frm)
if Recorder.is_disconn(inc):
disconnecteds = inc[1:]
replaying_node.nodestack._connsChanged(set(), disconnecteds)
if c_msgs:
incomings = Recorder.filter_incoming(c_msgs)
for inc in incomings:
msg, frm = to_bytes(inc[0]), to_bytes(inc[1])
replaying_node.clientstack._verifyAndAppend(msg, frm)
looper.run(replaying_node.prod())
return replaying_node
def create_recorder_for_test(tmpdir_factory, name):
storage = KeyValueStorageLeveldbIntKeys(
tmpdir_factory.mktemp('').strpath, name)
return Recorder(storage)