def __init__(self, val_weights, protocol=BlockchainProtocol, display=False):
self.validator_set = ValidatorSet(val_weights, protocol)
self.network = NoDelayNetwork(self.validator_set, protocol)
self.messages = dict()
self.plot_tool = protocol.PlotTool(
display,
False,
self.network.global_view,
self.validator_set
)
# Register token handlers.
self.handlers = dict()
self.register_handler('M', self.make_message)
self.register_handler('I', self.make_invalid)
self.register_handler('S', self.send_message)
self.register_handler('P', self.plot)
self.register_handler('SJ', self.send_and_justify)
self.register_handler('RR', self.round_robin)
self.register_handler('CE', self.check_estimate)
self.register_handler('CS', self.check_safe)
self.register_handler('CU', self.check_unsafe)
def __init__(self,
val_weights,
protocol=BlockchainProtocol,
display=False):
self.validator_set = ValidatorSet(val_weights, protocol)
self.display = display
self.network = Network(self.validator_set, protocol)
# This seems to be misnamed. Just generates starting blocks.
self.network.random_initialization()
self.plot_tool = PlotTool(display, False, 's')
self.blocks = dict()
self.blockchain = []
self.communications = []
self.block_fault_tolerance = dict()
# Register token handlers.
self.handlers = dict()
self.handlers['B'] = self.make_block
self.handlers['S'] = self.send_block
self.handlers['C'] = self.check_safety
self.handlers['U'] = self.no_safety
self.handlers['H'] = self.check_head_equals_block
self.handlers['RR'] = self.round_robin
self.handlers['R'] = self.report
def test_get_weight(weights, subset, expected_weight, view, message):
validator_set = ValidatorSet(weights, view, message)
validators = validator_set.get_validators_by_names(subset)
for validator in validator_set:
print("Name: {}, Weight: {}".format(validator.name, validator.weight))
assert round(utils.get_weight(validators), 2) == round(expected_weight, 2)
def __init__(
self,
num_validators,
execution_string,
messages_per_round,
display,
save,
plot_tool,
view_cls,
message_cls
):
self.global_validator_set = ValidatorSet(num_validators, view_cls, message_cls)
self.global_view = view_cls()
self.plot_tool = plot_tool(display, save, self.global_view, self.global_validator_set)
self.unexecuted = execution_string
self.executed = ''
self.messages_per_round = messages_per_round
self.messages_this_round = 0
self.messages = dict()
self.message_from_hash = dict()
self.message_name_from_hash = dict()
self.handlers = dict()
self.register_handler('M', self.make_message)
self.register_handler('S', self.send_message)
self.register_handler('SJ', self.send_and_justify)
def test_full_message_maker(weights, pairs):
validator_set = ValidatorSet(weights)
msg_gen = message_maker("full")
message_paths = msg_gen(validator_set)
for sender_name, receiver_name in pairs:
sender = validator_set.get_validator_by_name(sender_name)
receiver = validator_set.get_validator_by_name(receiver_name)
assert (sender, receiver) in message_paths
def test_get_weight(weights, expected_weight, validator_names):
validator_set = ValidatorSet(weights)
if expected_weight is None:
expected_weight = sum(weights.values())
if validator_names is None:
validators = validator_set.validators
else:
validators = validator_set.get_validators_by_names(validator_names)
assert round(utils.get_weight(validators), 2) == round(expected_weight, 2)
def test_estimator_picks_correct_estimate(weights, latest_estimates, estimate, empty_just):
validator_set = ValidatorSet(weights, IntegerProtocol)
latest_messages = dict()
for val_name in latest_estimates:
validator = validator_set.get_validator_by_name(val_name)
latest_messages[validator] = Bet(
latest_estimates[val_name], empty_just, validator, 1, 1
)
assert estimate == estimator.get_estimate_from_latest_messages(latest_messages)
def test_weight(weights, expected_weight, validator_names):
val_set = ValidatorSet(weights)
if expected_weight is None:
expected_weight = sum(weights.values())
if validator_names:
validators = val_set.get_validators_by_names(validator_names)
else:
validators = None
assert round(val_set.weight(validators), 2) == round(expected_weight, 2)
def test_get_validators_by_names(weights):
val_set = ValidatorSet(weights)
for i in range(1, len(weights)):
val_subsets = itertools.combinations(val_set, i)
for subset in val_subsets:
subset = {val for val in subset}
val_names = {validator.name for validator in subset}
returned_set = val_set.get_validators_by_names(val_names)
assert subset == returned_set
def test_get_validators_by_names(weights, view, message):
val_set = ValidatorSet(weights, view, message)
for i in range(1, len(weights)):
val_subsets = itertools.combinations(val_set, i)
for subset in val_subsets:
subset = {val for val in subset}
val_names = {validator.name for validator in subset}
returned_set = val_set.get_validators_by_names(val_names)
assert subset == returned_set
def test_weight(weights, expected_weight, validator_names, view, message):
val_set = ValidatorSet(weights, view, message)
if expected_weight is None:
expected_weight = sum(weights)
if validator_names:
validators = val_set.get_validators_by_names(validator_names)
else:
validators = None
assert round(val_set.weight(validators), 2) == round(expected_weight, 2)
def test_estimator_picks_correct_estimate(weights, latest_estimates, estimate,
empty_just):
validator_set = ValidatorSet(weights)
latest_messages = dict()
for val_name in latest_estimates:
validator = validator_set.get_validator_by_name(val_name)
latest_messages[validator] = Bet(latest_estimates[val_name],
empty_just, validator, 1, 1)
assert estimate == estimator.get_estimate_from_latest_messages(
latest_messages)
def test_full_message_maker(weights):
validator_set = ValidatorSet(weights)
msg_gen = message_maker("full")
message_makers = msg_gen(validator_set)
assert len(validator_set) == len(message_makers)
assert set(validator_set.validators) == set(message_makers)
def generate_random_gaussian_validator_set(protocol,
num_validators=5,
mu=60,
sigma=40,
min_weight=20):
"""Generates a random validator set."""
# Give the validators random weights in 0.,BIGINT;
# this "big" integer's job is to guarantee the "tie-breaking property"
# that no two subsets of validator's total weights are exactly equal.
# In prod, we will add a random epsilon to weights given by bond amounts,
# however, for the purposes of the current work, this will suffice.
BIGINT = 1000000000000
names = set(range(num_validators))
weights = {
i: max(min_weight, r.gauss(mu, sigma)) + 1.0 /
(BIGINT + r.uniform(0, 1)) + r.random()
for i in names
}
return ValidatorSet(weights, protocol)
def test_get_validator_by_name(weights):
val_set = ValidatorSet(weights)
for validator in val_set:
returned_val = val_set.get_validator_by_name(validator.name)
assert validator == returned_val
def test_new_validator_set(weights, view, message):
val_set = ValidatorSet(weights, view, message)
assert len(val_set.validators) == len(weights)
assert set([v.weight for v in val_set]) == set(weights)
def create(weights):
return ValidatorSet(weights, binary_class.View, binary_class.Message)
def test_get_validator_by_name(weights, view, message):
val_set = ValidatorSet(weights, view, message)
for validator in val_set:
returned_val = val_set.get_validator_by_name(validator.name)
assert validator == returned_val
def integer_validator_set(test_weight, integer_class):
return ValidatorSet(test_weight, integer_class.View, integer_class.Message)
def blockchain_validator_set(test_weight, blockchain_class):
return ValidatorSet(test_weight, blockchain_class.View,
blockchain_class.Message)
def test_in(weights):
val_set = ValidatorSet(weights)
for validator in val_set.validators:
assert validator in val_set
def test_validator_weights(weights, expected_weights, view, message):
val_set = ValidatorSet(weights, view, message)
assert val_set.validator_weights() == set(expected_weights)
def test_validator_weights(weights, expected_weights):
val_set = ValidatorSet(weights)
assert val_set.validator_weights() == set(expected_weights)
def test_new_validator_set(weights):
val_set = ValidatorSet(weights)
assert len(val_set.validators) == len(weights.keys())
assert set(map(lambda v: v.weight,
val_set.validators)) == set(weights.values())
def order_validator_set(test_weight, order_class):
return ValidatorSet(test_weight, order_class.View, order_class.Message)
class TestLangCBC(object):
"""Allows testing of simulation scenarios with small testing language."""
# Signal to py.test that TestLangCBC should not be discovered.
__test__ = False
TOKEN_PATTERN = '([A-Za-z]*)([0-9]*)([-]*)([A-Za-z0-9]*)'
def __init__(self,
val_weights,
protocol=BlockchainProtocol,
display=False):
self.validator_set = ValidatorSet(val_weights, protocol)
self.display = display
self.network = Network(self.validator_set, protocol)
# This seems to be misnamed. Just generates starting blocks.
self.network.random_initialization()
self.plot_tool = PlotTool(display, False, 's')
self.blocks = dict()
self.blockchain = []
self.communications = []
self.block_fault_tolerance = dict()
# Register token handlers.
self.handlers = dict()
self.handlers['B'] = self.make_block
self.handlers['S'] = self.send_block
self.handlers['C'] = self.check_safety
self.handlers['U'] = self.no_safety
self.handlers['H'] = self.check_head_equals_block
self.handlers['RR'] = self.round_robin
self.handlers['R'] = self.report
def _validate_validator(self, validator):
if validator not in self.validator_set:
raise ValueError('Validator {} does not exist'.format(validator))
def _validate_block_exists(self, block_name):
if block_name not in self.blocks:
raise ValueError('Block {} does not exist'.format(block_name))
def _validate_block_does_not_exist(self, block_name):
if block_name in self.blocks:
raise ValueError('Block {} already exists'.format(block_name))
def parse(self, test_string):
"""Parse the test_string, and run the test"""
for token in test_string.split(' '):
letter, validator, dash, name = re.match(self.TOKEN_PATTERN,
token).groups()
if letter + validator + dash + name != token:
raise ValueError("Bad token: %s" % token)
if validator != '':
try:
validator = self.validator_set.get_validator_by_name(
int(validator))
except KeyError:
raise ValueError(
"Validator {} does not exist".format(validator))
self.handlers[letter](validator, name)
def send_block(self, validator, block_name):
"""Send some validator a block."""
self._validate_validator(validator)
self._validate_block_exists(block_name)
block = self.blocks[block_name]
if block in validator.view.messages:
raise Exception('Validator {} has already seen block {}'.format(
validator, block_name))
self.network.propagate_message_to_validator(block, validator)
def make_block(self, validator, block_name):
"""Have some validator produce a block."""
self._validate_validator(validator)
self._validate_block_does_not_exist(block_name)
new_block = self.network.get_message_from_validator(validator)
if new_block.estimate is not None:
self.blockchain.append([new_block, new_block.estimate])
self.blocks[block_name] = new_block
def round_robin(self, validator, block_name):
"""Have each validator create a block in a perfect round robin."""
self._validate_validator(validator)
self._validate_block_does_not_exist(block_name)
# start round robin at validator speicied by validator in args
validators = self.validator_set.sorted_by_name()
start_index = validators.index(validator)
validators = validators[start_index:] + validators[:start_index]
for i in range(len(self.validator_set)):
if i == len(self.validator_set) - 1:
name = block_name
else:
name = r.random()
maker = validators[i]
receiver = validators[(i + 1) % len(validators)]
self.make_block(maker, name)
self.send_block(receiver, name)
def check_safety(self, validator, block_name):
"""Check that some validator detects safety on a block."""
self._validate_validator(validator)
self._validate_block_exists(block_name)
block = self.blocks[block_name]
validator.update_safe_estimates()
assert validator.view.last_finalized_block is None or \
not block.conflicts_with(validator.view.last_finalized_block), \
"Block {0} failed safety assert for validator-{1}".format(block_name, validator.name)
def no_safety(self, validator, block_name):
"""Check that some validator does not detect safety on a block."""
self._validate_validator(validator)
self._validate_block_exists(block_name)
block = self.blocks[block_name]
validator.update_safe_estimates()
#NOTE: This should never fail
assert validator.view.last_finalized_block is None or \
block.conflicts_with(validator.view.last_finalized_block), \
"Block {} failed no-safety assert".format(block_name)
def check_head_equals_block(self, validator, block_name):
"""Check some validators forkchoice is the correct block."""
self._validate_validator(validator)
self._validate_block_exists(block_name)
block = self.blocks[block_name]
head = validator.view.estimate()
assert block == head, "Validator {} does not have " \
"block {} at head".format(validator, block_name)
def report(self, num, name):
"""Display the view graph of the current global_view"""
assert num == name and num == '', "...no validator or number needed to report!"
if not self.display:
return
# Update the safe blocks!
tip = self.network.global_view.estimate()
while tip and self.block_fault_tolerance.get(
tip, 0) != len(self.validator_set) - 1:
oracle = CliqueOracle(tip, self.network.global_view,
self.validator_set)
fault_tolerance, num_node_ft = oracle.check_estimate_safety()
if fault_tolerance > 0:
self.block_fault_tolerance[tip] = num_node_ft
tip = tip.estimate
edgelist = []
best_chain = utils.build_chain(self.network.global_view.estimate(),
None)
edgelist.append(utils.edge(best_chain, 5, 'red', 'solid'))
for validator in self.validator_set:
chain = utils.build_chain(validator.my_latest_message(), None)
edgelist.append(utils.edge(chain, 2, 'blue', 'solid'))
edgelist.append(utils.edge(self.blockchain, 2, 'grey', 'solid'))
edgelist.append(utils.edge(self.communications, 1, 'black', 'dotted'))
message_labels = {}
for block in self.network.global_view.messages:
message_labels[block] = block.sequence_number
self.plot_tool.next_viewgraph(
self.network.global_view,
self.validator_set,
edges=edgelist,
message_colors=self.block_fault_tolerance,
message_labels=message_labels)
class Protocol(object):
def __init__(
self,
num_validators,
execution_string,
messages_per_round,
display,
save,
plot_tool,
view_cls,
message_cls
):
self.global_validator_set = ValidatorSet(num_validators, view_cls, message_cls)
self.global_view = view_cls()
self.plot_tool = plot_tool(display, save, self.global_view, self.global_validator_set)
self.unexecuted = execution_string
self.executed = ''
self.messages_per_round = messages_per_round
self.messages_this_round = 0
self.messages = dict()
self.message_from_hash = dict()
self.message_name_from_hash = dict()
self.handlers = dict()
self.register_handler('M', self.make_message)
self.register_handler('S', self.send_message)
self.register_handler('SJ', self.send_and_justify)
def register_handler(self, token, function):
"""Registers a function with a new token. Throws an error if already registered"""
if token in self.handlers:
raise KeyError('A function has been registered with that token')
self.handlers[token] = function
def register_message(self, message, name):
"""Register a new message with a new name"""
if name in self.messages:
raise KeyError("Message with name {} already exists".format(name))
if message.hash in self.message_from_hash:
raise KeyError("Message with hash {} already exists".format(message.hash))
self.messages[name] = message
self.message_from_hash[message.hash] = message
self.message_name_from_hash[message.hash] = name
self.global_view.add_messages([message])
self.plot_tool.update(new_messages=[message])
def make_message(self, validator, message_name, data):
"""Have a validator generate a new message"""
new_message = validator.make_new_message()
self.register_message(new_message, message_name)
def send_message(self, validator, message_name, data):
"""Send a message to a validator"""
message = self.messages[message_name]
validator.receive_messages(set([message]))
def send_and_justify(self, validator, message_name, data):
"""Send a message (and the messages in it's dependencies) to a validator"""
message = self.messages[message_name]
messages_to_send = self._messages_needed_to_justify(message, validator)
validator.receive_messages(messages_to_send)
def _messages_needed_to_justify(self, message, validator):
"""Returns the set of messages needed to justify a message to a validator"""
messages_needed = set([message])
current_message_hashes = set([message.hash])
while any(current_message_hashes):
next_hashes = set()
for message_hash in current_message_hashes:
message = self.message_from_hash[message_hash]
messages_needed.add(message)
for other_hash in message.justification.values():
if other_hash not in validator.view.justified_messages:
next_hashes.add(other_hash)
current_message_hashes = next_hashes
return messages_needed
def execute(self, additional_str=None):
"""Execute saved execution string as well as optional additional_str"""
if additional_str:
self.unexecuted += additional_str
for token in self.unexecuted.split():
comm, vali, name, data = Protocol.parse_token(token)
validator = self.global_validator_set.get_validator_by_name(int(vali))
self.handlers[comm](validator, name, data)
if comm == 'M':
self.messages_this_round += 1
if self.messages_this_round % self.messages_per_round == 0:
self.plot_tool.plot()
self.messages_this_round = 0
self.executed += self.unexecuted
self.unexecuted = ''
self.plot_tool.make_gif()
@staticmethod
def parse_token(token):
"""Break a token into a command, validator, name, and data"""
comm, _, vali, _, name, _, data = re.match(
TOKEN_PATTERN, token
).groups()
if data:
if comm + '-' + vali + '-' + name + '-' + data != token:
raise ValueError("Bad token: {}".format(token))
else:
if comm + '-' + vali + '-' + name != token:
raise ValueError("Bad token: {}".format(token))
return comm, vali, name, data
def test_in(weights, view, message):
val_set = ValidatorSet(weights, view, message)
for validator in val_set.validators:
assert validator in val_set
def test_len(weights):
val_set = ValidatorSet(weights)
assert len(val_set) == len(weights)
def test_len(weights, view, message):
val_set = ValidatorSet(weights, view, message)
assert len(val_set) == len(weights)
def test_validator_names(weights, expected_names):
val_set = ValidatorSet(weights)
assert val_set.validator_names() == set(expected_names)
def weights_generator():
return ValidatorSet(jitter_weights, protocol)
def binary_validator_set(test_weight, binary_class):
return ValidatorSet(test_weight, binary_class.View, binary_class.Message)
class StateLanguage(object):
"""Allows testing of simulation scenarios with small testing language."""
TOKEN_PATTERN = '([A-Za-z]*)([0-9]*)([-]*)([A-Za-z0-9]*)([\\{A-Za-z,}]*)'
def __init__(self, val_weights, protocol=BlockchainProtocol, display=False):
self.validator_set = ValidatorSet(val_weights, protocol)
self.network = NoDelayNetwork(self.validator_set, protocol)
self.messages = dict()
self.plot_tool = protocol.PlotTool(
display,
False,
self.network.global_view,
self.validator_set
)
# Register token handlers.
self.handlers = dict()
self.register_handler('M', self.make_message)
self.register_handler('I', self.make_invalid)
self.register_handler('S', self.send_message)
self.register_handler('P', self.plot)
self.register_handler('SJ', self.send_and_justify)
self.register_handler('RR', self.round_robin)
self.register_handler('CE', self.check_estimate)
self.register_handler('CS', self.check_safe)
self.register_handler('CU', self.check_unsafe)
def register_handler(self, token, function):
"""Registers a function with a new token. Throws an error if already registered"""
if token in self.handlers:
raise KeyError('A function has been registered with that token')
self.handlers[token] = function
def make_message(self, validator, message_name, messages_to_hide=None):
"""Have a validator generate a new message"""
self.require_message_not_exists(message_name)
#NOTE: Once validators have the ability to lie about their view, hide messages_to_hide!
new_message = validator.make_new_message()
self.network.global_view.add_messages(
set([new_message])
)
self.plot_tool.update([new_message])
self.messages[message_name] = new_message
def send_message(self, validator, message_name):
"""Send a message to a specific validator"""
self.require_message_exists(message_name)
message = self.messages[message_name]
self._propagate_message_to_validator(validator, message)
def make_invalid(self, validator, message_name):
"""TODO: Implement this when validators can make/handle invalid messages"""
raise NotImplementedError
def send_and_justify(self, validator, message_name):
self.require_message_exists(message_name)
message = self.messages[message_name]
self._propagate_message_to_validator(validator, message)
messages_to_send = self._messages_needed_to_justify(message, validator)
for message in messages_to_send:
self._propagate_message_to_validator(validator, message)
assert self.messages[message_name].hash in validator.view.justified_messages
def round_robin(self, validator, message_name):
"""Have each validator create a message in a perfect round robin."""
self.require_message_not_exists(message_name)
# start round robin at validator specified by validator in args
validators = self.validator_set.sorted_by_name()
start_index = validators.index(validator)
validators = validators[start_index:] + validators[:start_index]
for i in range(len(self.validator_set)):
if i == len(self.validator_set) - 1:
name = message_name
else:
name = r.random()
maker = validators[i]
receiver = validators[(i + 1) % len(validators)]
self.make_message(maker, name)
self.send_and_justify(receiver, name)
def plot(self):
"""Display or save a viewgraph"""
self.plot_tool.plot()
def check_estimate(self, validator, estimate):
"""Must be implemented by child class"""
raise NotImplementedError
def check_safe(self, validator, estimate):
"""Must be implemented by child class"""
raise NotImplementedError
def check_unsafe(self, validator, estimate):
"""Must be implemented by child class"""
raise NotImplementedError
def require_message_exists(self, message_name):
"""Throws an error if message_name does not exist"""
if message_name not in self.messages:
raise ValueError('Block {} does not exist'.format(message_name))
def require_message_not_exists(self, message_name):
"""Throws an error if message_name does not exist"""
if message_name in self.messages:
raise ValueError('Block {} already exists'.format(message_name))
def _propagate_message_to_validator(self, validator, message):
self.network.send(validator, message)
received_message = self.network.receive(validator)
if received_message:
validator.receive_messages(set([received_message]))
def _messages_needed_to_justify(self, message, validator):
"""Returns the set of messages needed to justify a message to a validator"""
messages_needed = set()
current_message_hashes = set()
for message_hash in message.justification.values():
if message_hash not in validator.view.pending_messages and \
message_hash not in validator.view.justified_messages:
current_message_hashes.add(message_hash)
while any(current_message_hashes):
next_hashes = set()
for message_hash in current_message_hashes:
message = self.network.global_view.justified_messages[message_hash]
messages_needed.add(message)
for other_hash in message.justification.values():
if other_hash not in validator.view.pending_messages and \
other_hash not in validator.view.justified_messages:
next_hashes.add(other_hash)
current_message_hashes = next_hashes
return messages_needed
def parse(self, protocol_state_string):
"""Parse the state string!"""
for token in protocol_state_string.split():
letter, validator, message = self.parse_token(token)
if letter == 'P':
self.plot()
else:
validator = self.validator_set.get_validator_by_name(int(validator))
self.handlers[letter](validator, message)
def parse_token(self, token):
letter, validator, dash, message, removed_message_names = re.match(
self.TOKEN_PATTERN, token
).groups()
if letter + validator + dash + message + removed_message_names != token:
raise ValueError("Bad token: %s" % token)
return letter, validator, message
def create(weight):
return ValidatorSet(weight, order_class.View, order_class.Message)
