def test_append():
slt = SortedListWithKey(load=4, key=modulo)
slt.append(0)
for val in range(1, 10):
slt.append(val)
slt._check()
def test_append():
slt = SortedListWithKey(load=17, key=negate)
slt.append(1000)
for val in range(999, -1, -1):
slt.append(val)
slt._check()
def test_append():
slt = SortedListWithKey(load=4, key=modulo, value_orderable=False)
slt.append(0)
for val in range(1, 10):
slt.append(val)
slt._check()
def test_append():
slt = SortedListWithKey(load=4, key=modulo, value_orderable=False)
slt.append(0)
for val in range(1, 10):
slt.append(val)
slt._check()
def test_append():
slt = SortedListWithKey(load=17, key=negate)
slt.append(1000)
for val in range(999, -1, -1):
slt.append(val)
slt._check()
def test_append():
slt = SortedListWithKey(load=4, key=modulo)
slt.append(0)
for val in range(1, 10):
slt.append(val)
slt._check()
def test_getitem():
random.seed(0)
slt = SortedListWithKey(load=17, key=modulo)
slt.append(5)
slt._build_index()
slt._check()
slt.clear()
lst = list(random.random() for rpt in range(100))
slt.update(lst)
lst.sort(key=modulo)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def test_getitem():
random.seed(0)
slt = SortedListWithKey(load=17, key=modulo)
slt.append(5)
slt._build_index()
slt._check()
slt.clear()
lst = list(random.random() for rpt in range(100))
slt.update(lst)
lst.sort(key=modulo)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def test_getitem():
random.seed(0)
slt = SortedListWithKey(load=17, key=negate)
slt.append(5)
assert slt[0] == 5
slt.clear()
lst = list()
for rpt in range(100):
val = random.random()
slt.add(val)
lst.append(val)
lst.sort(reverse=True)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def test_getitem():
random.seed(0)
slt = SortedListWithKey(load=17, key=negate)
slt.append(5)
assert slt[0] == 5
slt.clear()
lst = list()
for rpt in range(100):
val = random.random()
slt.add(val)
lst.append(val)
lst.sort(reverse=True)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def parse_mlf(fname, divisor):
all_label = dict()
with fileinput.input((fname, ), mode='r') as f:
f.readline()
for line in f:
if line.startswith('"'):
audio_id = line.strip('"\n')[:-4]
audio_id = audio_id.split('/')[-1]
labels = SortedListWithKey(key=lambda x: x[0])
continue
if line[0] == '.':
all_label[audio_id] = labels
continue
line = line.strip().split()
if len(line) < 3:
print('Warning: Invalid line', line)
continue
label = (int(line[0]) // divisor, int(line[1]) // divisor, line[2])
#label = line[2]
labels.append(label)
return all_label
class ConsensusSharedData:
"""
This is a 3PC-state shared between Ordering, Checkpoint and ViewChange services.
TODO: Consider depending on audit ledger
TODO: Consider adding persistent local storage for 3PC certificates
TODO: Restore primary name from audit ledger instead of passing through constructor
"""
def __init__(self,
name: str,
validators: List[str],
inst_id: int,
is_master: bool = True):
self._name = name
self.inst_id = inst_id
self.view_no = 0
self.waiting_for_new_view = False
self.primaries = []
self.is_master = is_master
self.legacy_vc_in_progress = False
self.requests = Requests()
self.last_ordered_3pc = (0, 0)
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
# TODO: Replace this by read-only property which uses primaries and inst_id
self.primary_name = None
# seqNoEnd of the last stabilized checkpoint
self.stable_checkpoint = 0
# Checkpoint messages which the current node sent.
# TODO: Replace sorted list with dict
self.checkpoints = SortedListWithKey(
key=lambda checkpoint: checkpoint.seqNoEnd)
self.checkpoints.append(self.initial_checkpoint)
# List of BatchIDs of PrePrepare messages for which quorum of Prepare messages is not reached yet
self.preprepared = [] # type: List[BatchID]
# List of BatchIDs of PrePrepare messages for which quorum of Prepare messages is reached
self.prepared = [] # type: List[BatchID]
self._validators = None
self.quorums = None
self.view_change_votes = ViewChangeVotesForView(
Quorums(len(validators)))
# a list of validator node names ordered by rank (historical order of adding)
self.set_validators(validators)
self.low_watermark = 0
self.log_size = getConfig().LOG_SIZE
self.high_watermark = self.low_watermark + self.log_size
self.pp_seq_no = 0
self.node_mode = Mode.starting
# ToDo: it should be set in view_change_service before view_change starting
# 3 phase key for the last prepared certificate before view change
# started, applicable only to master instance
self.legacy_last_prepared_before_view_change = None
self.prev_view_prepare_cert = None
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is the received PRE-PREPARE
self.sent_preprepares = SortedDict(lambda k: (k[0], k[1]))
# type: Dict[Tuple[int, int], PrePrepare]
# Dictionary of all Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> ((identifier, reqId), {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]
self.commits = Commits()
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]
# Tracks for which keys PRE-PREPAREs have been requested.
# Cleared in `gc`
self.requested_pre_prepares = {}
@property
def name(self) -> str:
return self._name
def set_validators(self, validators: List[str]):
self._validators = validators
self.quorums = Quorums(len(validators))
self.view_change_votes.update_quorums(self.quorums)
@property
def validators(self) -> List[str]:
"""
List of validator nodes aliases
"""
return self._validators
@property
def is_primary(self) -> Optional[bool]:
"""
TODO: It would be much more clear and easy to use if this returned just bool.
Returns is replica primary for this instance.
If primary name is not defined yet, returns None
"""
return None if self.primary_name is None else self.primary_name == self.name
@property
def is_participating(self):
return self.node_mode == Mode.participating
@property
def is_synced(self):
return Mode.is_done_syncing(self.node_mode)
@property
def total_nodes(self):
return len(self.validators)
@property
def initial_checkpoint(self):
return Checkpoint(instId=self.inst_id,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=None)
@property
def last_checkpoint(self) -> Checkpoint:
return self.checkpoints[-1]
def test_append_valueerror():
slt = SortedListWithKey(range(100), key=modulo)
slt.append(5)
class ConsensusSharedData:
"""
This is a 3PC-state shared between Ordering, Checkpoint and ViewChange services.
TODO: Consider depending on audit ledger
TODO: Consider adding persistent local storage for 3PC certificates
TODO: Restore primary name from audit ledger instead of passing through constructor
"""
def __init__(self,
name: str,
validators: List[str],
inst_id: int,
is_master: bool = True):
self._name = name
self.inst_id = inst_id
self.view_no = 0
self.waiting_for_new_view = False
self.primaries = []
self.is_master = is_master
self.legacy_vc_in_progress = False
self.requests = Requests()
self.last_ordered_3pc = (0, 0)
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
# TODO: Replace this by read-only property which uses primaries and inst_id
self.primary_name = None
# seqNoEnd of the last stabilized checkpoint
self.stable_checkpoint = 0
# Checkpoint messages which the current node sent.
# TODO: Replace sorted list with dict
self.checkpoints = SortedListWithKey(
key=lambda checkpoint: checkpoint.seqNoEnd)
self.checkpoints.append(self.initial_checkpoint)
# List of BatchIDs of PrePrepare messages for which quorum of Prepare messages is not reached yet
self.preprepared = [] # type: List[BatchID]
# List of BatchIDs of PrePrepare messages for which quorum of Prepare messages is reached
self.prepared = [] # type: List[BatchID]
self._validators = None
self.quorums = None
# a list of validator node names ordered by rank (historical order of adding)
self.set_validators(validators)
self.low_watermark = 0
self.log_size = getConfig().LOG_SIZE
self.high_watermark = self.low_watermark + self.log_size
self.pp_seq_no = 0
self.node_mode = Mode.starting
# ToDo: it should be set in view_change_service before view_change starting
# 3 phase key for the last prepared certificate before view change
# started, applicable only to master instance
self.legacy_last_prepared_before_view_change = None
@property
def name(self) -> str:
return self._name
def set_validators(self, validators: List[str]):
self._validators = validators
self.quorums = Quorums(len(validators))
@property
def validators(self) -> List[str]:
"""
List of validator nodes aliases
"""
return self._validators
@property
def is_primary(self) -> Optional[bool]:
"""
TODO: It would be much more clear and easy to use if this returned just bool.
Returns is replica primary for this instance.
If primary name is not defined yet, returns None
"""
return None if self.primary_name is None else self.primary_name == self.name
@property
def is_participating(self):
return self.node_mode == Mode.participating
@property
def is_synced(self):
return Mode.is_done_syncing(self.node_mode)
@property
def total_nodes(self):
return len(self.validators)
@property
def initial_checkpoint(self):
return Checkpoint(instId=self.inst_id,
viewNo=0,
seqNoStart=0,
seqNoEnd=0,
digest=None)
@property
def last_checkpoint(self) -> Checkpoint:
return self.checkpoints[-1]
def test_append_valueerror():
slt = SortedListWithKey(range(100), key=modulo, value_orderable=False)
slt.append(5)
def test_append_valueerror():
slt = SortedListWithKey(range(100), key=modulo)
slt.append(5)
def test_append_valueerror():
slt = SortedListWithKey(range(100), key=modulo, value_orderable=False)
slt.append(5)