def __init__(self, val_weights, protocol=BlockchainProtocol, display=False): self.validator_set = ValidatorSet(val_weights, protocol) self.network = NoDelayNetwork(self.validator_set, protocol) self.messages = dict() self.plot_tool = protocol.PlotTool( display, False, self.network.global_view, self.validator_set ) # Register token handlers. self.handlers = dict() self.register_handler('M', self.make_message) self.register_handler('I', self.make_invalid) self.register_handler('S', self.send_message) self.register_handler('P', self.plot) self.register_handler('SJ', self.send_and_justify) self.register_handler('RR', self.round_robin) self.register_handler('CE', self.check_estimate) self.register_handler('CS', self.check_safe) self.register_handler('CU', self.check_unsafe)
def __init__(self, val_weights, protocol=BlockchainProtocol, display=False): self.validator_set = ValidatorSet(val_weights, protocol) self.display = display self.network = Network(self.validator_set, protocol) # This seems to be misnamed. Just generates starting blocks. self.network.random_initialization() self.plot_tool = PlotTool(display, False, 's') self.blocks = dict() self.blockchain = [] self.communications = [] self.block_fault_tolerance = dict() # Register token handlers. self.handlers = dict() self.handlers['B'] = self.make_block self.handlers['S'] = self.send_block self.handlers['C'] = self.check_safety self.handlers['U'] = self.no_safety self.handlers['H'] = self.check_head_equals_block self.handlers['RR'] = self.round_robin self.handlers['R'] = self.report
def test_get_weight(weights, subset, expected_weight, view, message): validator_set = ValidatorSet(weights, view, message) validators = validator_set.get_validators_by_names(subset) for validator in validator_set: print("Name: {}, Weight: {}".format(validator.name, validator.weight)) assert round(utils.get_weight(validators), 2) == round(expected_weight, 2)
def __init__( self, num_validators, execution_string, messages_per_round, display, save, plot_tool, view_cls, message_cls ): self.global_validator_set = ValidatorSet(num_validators, view_cls, message_cls) self.global_view = view_cls() self.plot_tool = plot_tool(display, save, self.global_view, self.global_validator_set) self.unexecuted = execution_string self.executed = '' self.messages_per_round = messages_per_round self.messages_this_round = 0 self.messages = dict() self.message_from_hash = dict() self.message_name_from_hash = dict() self.handlers = dict() self.register_handler('M', self.make_message) self.register_handler('S', self.send_message) self.register_handler('SJ', self.send_and_justify)
def test_full_message_maker(weights, pairs): validator_set = ValidatorSet(weights) msg_gen = message_maker("full") message_paths = msg_gen(validator_set) for sender_name, receiver_name in pairs: sender = validator_set.get_validator_by_name(sender_name) receiver = validator_set.get_validator_by_name(receiver_name) assert (sender, receiver) in message_paths
def test_get_weight(weights, expected_weight, validator_names): validator_set = ValidatorSet(weights) if expected_weight is None: expected_weight = sum(weights.values()) if validator_names is None: validators = validator_set.validators else: validators = validator_set.get_validators_by_names(validator_names) assert round(utils.get_weight(validators), 2) == round(expected_weight, 2)
def test_estimator_picks_correct_estimate(weights, latest_estimates, estimate, empty_just): validator_set = ValidatorSet(weights, IntegerProtocol) latest_messages = dict() for val_name in latest_estimates: validator = validator_set.get_validator_by_name(val_name) latest_messages[validator] = Bet( latest_estimates[val_name], empty_just, validator, 1, 1 ) assert estimate == estimator.get_estimate_from_latest_messages(latest_messages)
def test_weight(weights, expected_weight, validator_names): val_set = ValidatorSet(weights) if expected_weight is None: expected_weight = sum(weights.values()) if validator_names: validators = val_set.get_validators_by_names(validator_names) else: validators = None assert round(val_set.weight(validators), 2) == round(expected_weight, 2)
def test_get_validators_by_names(weights): val_set = ValidatorSet(weights) for i in range(1, len(weights)): val_subsets = itertools.combinations(val_set, i) for subset in val_subsets: subset = {val for val in subset} val_names = {validator.name for validator in subset} returned_set = val_set.get_validators_by_names(val_names) assert subset == returned_set
def test_get_validators_by_names(weights, view, message): val_set = ValidatorSet(weights, view, message) for i in range(1, len(weights)): val_subsets = itertools.combinations(val_set, i) for subset in val_subsets: subset = {val for val in subset} val_names = {validator.name for validator in subset} returned_set = val_set.get_validators_by_names(val_names) assert subset == returned_set
def test_weight(weights, expected_weight, validator_names, view, message): val_set = ValidatorSet(weights, view, message) if expected_weight is None: expected_weight = sum(weights) if validator_names: validators = val_set.get_validators_by_names(validator_names) else: validators = None assert round(val_set.weight(validators), 2) == round(expected_weight, 2)
def test_estimator_picks_correct_estimate(weights, latest_estimates, estimate, empty_just): validator_set = ValidatorSet(weights) latest_messages = dict() for val_name in latest_estimates: validator = validator_set.get_validator_by_name(val_name) latest_messages[validator] = Bet(latest_estimates[val_name], empty_just, validator, 1, 1) assert estimate == estimator.get_estimate_from_latest_messages( latest_messages)
def test_full_message_maker(weights): validator_set = ValidatorSet(weights) msg_gen = message_maker("full") message_makers = msg_gen(validator_set) assert len(validator_set) == len(message_makers) assert set(validator_set.validators) == set(message_makers)
def generate_random_gaussian_validator_set(protocol, num_validators=5, mu=60, sigma=40, min_weight=20): """Generates a random validator set.""" # Give the validators random weights in 0.,BIGINT; # this "big" integer's job is to guarantee the "tie-breaking property" # that no two subsets of validator's total weights are exactly equal. # In prod, we will add a random epsilon to weights given by bond amounts, # however, for the purposes of the current work, this will suffice. BIGINT = 1000000000000 names = set(range(num_validators)) weights = { i: max(min_weight, r.gauss(mu, sigma)) + 1.0 / (BIGINT + r.uniform(0, 1)) + r.random() for i in names } return ValidatorSet(weights, protocol)
def test_get_validator_by_name(weights): val_set = ValidatorSet(weights) for validator in val_set: returned_val = val_set.get_validator_by_name(validator.name) assert validator == returned_val
def test_new_validator_set(weights, view, message): val_set = ValidatorSet(weights, view, message) assert len(val_set.validators) == len(weights) assert set([v.weight for v in val_set]) == set(weights)
def create(weights): return ValidatorSet(weights, binary_class.View, binary_class.Message)
def test_get_validator_by_name(weights, view, message): val_set = ValidatorSet(weights, view, message) for validator in val_set: returned_val = val_set.get_validator_by_name(validator.name) assert validator == returned_val
def integer_validator_set(test_weight, integer_class): return ValidatorSet(test_weight, integer_class.View, integer_class.Message)
def blockchain_validator_set(test_weight, blockchain_class): return ValidatorSet(test_weight, blockchain_class.View, blockchain_class.Message)
def test_in(weights): val_set = ValidatorSet(weights) for validator in val_set.validators: assert validator in val_set
def test_validator_weights(weights, expected_weights, view, message): val_set = ValidatorSet(weights, view, message) assert val_set.validator_weights() == set(expected_weights)
def test_validator_weights(weights, expected_weights): val_set = ValidatorSet(weights) assert val_set.validator_weights() == set(expected_weights)
def test_new_validator_set(weights): val_set = ValidatorSet(weights) assert len(val_set.validators) == len(weights.keys()) assert set(map(lambda v: v.weight, val_set.validators)) == set(weights.values())
def order_validator_set(test_weight, order_class): return ValidatorSet(test_weight, order_class.View, order_class.Message)
class TestLangCBC(object): """Allows testing of simulation scenarios with small testing language.""" # Signal to py.test that TestLangCBC should not be discovered. __test__ = False TOKEN_PATTERN = '([A-Za-z]*)([0-9]*)([-]*)([A-Za-z0-9]*)' def __init__(self, val_weights, protocol=BlockchainProtocol, display=False): self.validator_set = ValidatorSet(val_weights, protocol) self.display = display self.network = Network(self.validator_set, protocol) # This seems to be misnamed. Just generates starting blocks. self.network.random_initialization() self.plot_tool = PlotTool(display, False, 's') self.blocks = dict() self.blockchain = [] self.communications = [] self.block_fault_tolerance = dict() # Register token handlers. self.handlers = dict() self.handlers['B'] = self.make_block self.handlers['S'] = self.send_block self.handlers['C'] = self.check_safety self.handlers['U'] = self.no_safety self.handlers['H'] = self.check_head_equals_block self.handlers['RR'] = self.round_robin self.handlers['R'] = self.report def _validate_validator(self, validator): if validator not in self.validator_set: raise ValueError('Validator {} does not exist'.format(validator)) def _validate_block_exists(self, block_name): if block_name not in self.blocks: raise ValueError('Block {} does not exist'.format(block_name)) def _validate_block_does_not_exist(self, block_name): if block_name in self.blocks: raise ValueError('Block {} already exists'.format(block_name)) def parse(self, test_string): """Parse the test_string, and run the test""" for token in test_string.split(' '): letter, validator, dash, name = re.match(self.TOKEN_PATTERN, token).groups() if letter + validator + dash + name != token: raise ValueError("Bad token: %s" % token) if validator != '': try: validator = self.validator_set.get_validator_by_name( int(validator)) except KeyError: raise ValueError( "Validator {} does not exist".format(validator)) self.handlers[letter](validator, name) def send_block(self, validator, block_name): """Send some validator a block.""" self._validate_validator(validator) self._validate_block_exists(block_name) block = self.blocks[block_name] if block in validator.view.messages: raise Exception('Validator {} has already seen block {}'.format( validator, block_name)) self.network.propagate_message_to_validator(block, validator) def make_block(self, validator, block_name): """Have some validator produce a block.""" self._validate_validator(validator) self._validate_block_does_not_exist(block_name) new_block = self.network.get_message_from_validator(validator) if new_block.estimate is not None: self.blockchain.append([new_block, new_block.estimate]) self.blocks[block_name] = new_block def round_robin(self, validator, block_name): """Have each validator create a block in a perfect round robin.""" self._validate_validator(validator) self._validate_block_does_not_exist(block_name) # start round robin at validator speicied by validator in args validators = self.validator_set.sorted_by_name() start_index = validators.index(validator) validators = validators[start_index:] + validators[:start_index] for i in range(len(self.validator_set)): if i == len(self.validator_set) - 1: name = block_name else: name = r.random() maker = validators[i] receiver = validators[(i + 1) % len(validators)] self.make_block(maker, name) self.send_block(receiver, name) def check_safety(self, validator, block_name): """Check that some validator detects safety on a block.""" self._validate_validator(validator) self._validate_block_exists(block_name) block = self.blocks[block_name] validator.update_safe_estimates() assert validator.view.last_finalized_block is None or \ not block.conflicts_with(validator.view.last_finalized_block), \ "Block {0} failed safety assert for validator-{1}".format(block_name, validator.name) def no_safety(self, validator, block_name): """Check that some validator does not detect safety on a block.""" self._validate_validator(validator) self._validate_block_exists(block_name) block = self.blocks[block_name] validator.update_safe_estimates() #NOTE: This should never fail assert validator.view.last_finalized_block is None or \ block.conflicts_with(validator.view.last_finalized_block), \ "Block {} failed no-safety assert".format(block_name) def check_head_equals_block(self, validator, block_name): """Check some validators forkchoice is the correct block.""" self._validate_validator(validator) self._validate_block_exists(block_name) block = self.blocks[block_name] head = validator.view.estimate() assert block == head, "Validator {} does not have " \ "block {} at head".format(validator, block_name) def report(self, num, name): """Display the view graph of the current global_view""" assert num == name and num == '', "...no validator or number needed to report!" if not self.display: return # Update the safe blocks! tip = self.network.global_view.estimate() while tip and self.block_fault_tolerance.get( tip, 0) != len(self.validator_set) - 1: oracle = CliqueOracle(tip, self.network.global_view, self.validator_set) fault_tolerance, num_node_ft = oracle.check_estimate_safety() if fault_tolerance > 0: self.block_fault_tolerance[tip] = num_node_ft tip = tip.estimate edgelist = [] best_chain = utils.build_chain(self.network.global_view.estimate(), None) edgelist.append(utils.edge(best_chain, 5, 'red', 'solid')) for validator in self.validator_set: chain = utils.build_chain(validator.my_latest_message(), None) edgelist.append(utils.edge(chain, 2, 'blue', 'solid')) edgelist.append(utils.edge(self.blockchain, 2, 'grey', 'solid')) edgelist.append(utils.edge(self.communications, 1, 'black', 'dotted')) message_labels = {} for block in self.network.global_view.messages: message_labels[block] = block.sequence_number self.plot_tool.next_viewgraph( self.network.global_view, self.validator_set, edges=edgelist, message_colors=self.block_fault_tolerance, message_labels=message_labels)
class Protocol(object): def __init__( self, num_validators, execution_string, messages_per_round, display, save, plot_tool, view_cls, message_cls ): self.global_validator_set = ValidatorSet(num_validators, view_cls, message_cls) self.global_view = view_cls() self.plot_tool = plot_tool(display, save, self.global_view, self.global_validator_set) self.unexecuted = execution_string self.executed = '' self.messages_per_round = messages_per_round self.messages_this_round = 0 self.messages = dict() self.message_from_hash = dict() self.message_name_from_hash = dict() self.handlers = dict() self.register_handler('M', self.make_message) self.register_handler('S', self.send_message) self.register_handler('SJ', self.send_and_justify) def register_handler(self, token, function): """Registers a function with a new token. Throws an error if already registered""" if token in self.handlers: raise KeyError('A function has been registered with that token') self.handlers[token] = function def register_message(self, message, name): """Register a new message with a new name""" if name in self.messages: raise KeyError("Message with name {} already exists".format(name)) if message.hash in self.message_from_hash: raise KeyError("Message with hash {} already exists".format(message.hash)) self.messages[name] = message self.message_from_hash[message.hash] = message self.message_name_from_hash[message.hash] = name self.global_view.add_messages([message]) self.plot_tool.update(new_messages=[message]) def make_message(self, validator, message_name, data): """Have a validator generate a new message""" new_message = validator.make_new_message() self.register_message(new_message, message_name) def send_message(self, validator, message_name, data): """Send a message to a validator""" message = self.messages[message_name] validator.receive_messages(set([message])) def send_and_justify(self, validator, message_name, data): """Send a message (and the messages in it's dependencies) to a validator""" message = self.messages[message_name] messages_to_send = self._messages_needed_to_justify(message, validator) validator.receive_messages(messages_to_send) def _messages_needed_to_justify(self, message, validator): """Returns the set of messages needed to justify a message to a validator""" messages_needed = set([message]) current_message_hashes = set([message.hash]) while any(current_message_hashes): next_hashes = set() for message_hash in current_message_hashes: message = self.message_from_hash[message_hash] messages_needed.add(message) for other_hash in message.justification.values(): if other_hash not in validator.view.justified_messages: next_hashes.add(other_hash) current_message_hashes = next_hashes return messages_needed def execute(self, additional_str=None): """Execute saved execution string as well as optional additional_str""" if additional_str: self.unexecuted += additional_str for token in self.unexecuted.split(): comm, vali, name, data = Protocol.parse_token(token) validator = self.global_validator_set.get_validator_by_name(int(vali)) self.handlers[comm](validator, name, data) if comm == 'M': self.messages_this_round += 1 if self.messages_this_round % self.messages_per_round == 0: self.plot_tool.plot() self.messages_this_round = 0 self.executed += self.unexecuted self.unexecuted = '' self.plot_tool.make_gif() @staticmethod def parse_token(token): """Break a token into a command, validator, name, and data""" comm, _, vali, _, name, _, data = re.match( TOKEN_PATTERN, token ).groups() if data: if comm + '-' + vali + '-' + name + '-' + data != token: raise ValueError("Bad token: {}".format(token)) else: if comm + '-' + vali + '-' + name != token: raise ValueError("Bad token: {}".format(token)) return comm, vali, name, data
def test_in(weights, view, message): val_set = ValidatorSet(weights, view, message) for validator in val_set.validators: assert validator in val_set
def test_len(weights): val_set = ValidatorSet(weights) assert len(val_set) == len(weights)
def test_len(weights, view, message): val_set = ValidatorSet(weights, view, message) assert len(val_set) == len(weights)
def test_validator_names(weights, expected_names): val_set = ValidatorSet(weights) assert val_set.validator_names() == set(expected_names)
def weights_generator(): return ValidatorSet(jitter_weights, protocol)
def binary_validator_set(test_weight, binary_class): return ValidatorSet(test_weight, binary_class.View, binary_class.Message)
class StateLanguage(object): """Allows testing of simulation scenarios with small testing language.""" TOKEN_PATTERN = '([A-Za-z]*)([0-9]*)([-]*)([A-Za-z0-9]*)([\\{A-Za-z,}]*)' def __init__(self, val_weights, protocol=BlockchainProtocol, display=False): self.validator_set = ValidatorSet(val_weights, protocol) self.network = NoDelayNetwork(self.validator_set, protocol) self.messages = dict() self.plot_tool = protocol.PlotTool( display, False, self.network.global_view, self.validator_set ) # Register token handlers. self.handlers = dict() self.register_handler('M', self.make_message) self.register_handler('I', self.make_invalid) self.register_handler('S', self.send_message) self.register_handler('P', self.plot) self.register_handler('SJ', self.send_and_justify) self.register_handler('RR', self.round_robin) self.register_handler('CE', self.check_estimate) self.register_handler('CS', self.check_safe) self.register_handler('CU', self.check_unsafe) def register_handler(self, token, function): """Registers a function with a new token. Throws an error if already registered""" if token in self.handlers: raise KeyError('A function has been registered with that token') self.handlers[token] = function def make_message(self, validator, message_name, messages_to_hide=None): """Have a validator generate a new message""" self.require_message_not_exists(message_name) #NOTE: Once validators have the ability to lie about their view, hide messages_to_hide! new_message = validator.make_new_message() self.network.global_view.add_messages( set([new_message]) ) self.plot_tool.update([new_message]) self.messages[message_name] = new_message def send_message(self, validator, message_name): """Send a message to a specific validator""" self.require_message_exists(message_name) message = self.messages[message_name] self._propagate_message_to_validator(validator, message) def make_invalid(self, validator, message_name): """TODO: Implement this when validators can make/handle invalid messages""" raise NotImplementedError def send_and_justify(self, validator, message_name): self.require_message_exists(message_name) message = self.messages[message_name] self._propagate_message_to_validator(validator, message) messages_to_send = self._messages_needed_to_justify(message, validator) for message in messages_to_send: self._propagate_message_to_validator(validator, message) assert self.messages[message_name].hash in validator.view.justified_messages def round_robin(self, validator, message_name): """Have each validator create a message in a perfect round robin.""" self.require_message_not_exists(message_name) # start round robin at validator specified by validator in args validators = self.validator_set.sorted_by_name() start_index = validators.index(validator) validators = validators[start_index:] + validators[:start_index] for i in range(len(self.validator_set)): if i == len(self.validator_set) - 1: name = message_name else: name = r.random() maker = validators[i] receiver = validators[(i + 1) % len(validators)] self.make_message(maker, name) self.send_and_justify(receiver, name) def plot(self): """Display or save a viewgraph""" self.plot_tool.plot() def check_estimate(self, validator, estimate): """Must be implemented by child class""" raise NotImplementedError def check_safe(self, validator, estimate): """Must be implemented by child class""" raise NotImplementedError def check_unsafe(self, validator, estimate): """Must be implemented by child class""" raise NotImplementedError def require_message_exists(self, message_name): """Throws an error if message_name does not exist""" if message_name not in self.messages: raise ValueError('Block {} does not exist'.format(message_name)) def require_message_not_exists(self, message_name): """Throws an error if message_name does not exist""" if message_name in self.messages: raise ValueError('Block {} already exists'.format(message_name)) def _propagate_message_to_validator(self, validator, message): self.network.send(validator, message) received_message = self.network.receive(validator) if received_message: validator.receive_messages(set([received_message])) def _messages_needed_to_justify(self, message, validator): """Returns the set of messages needed to justify a message to a validator""" messages_needed = set() current_message_hashes = set() for message_hash in message.justification.values(): if message_hash not in validator.view.pending_messages and \ message_hash not in validator.view.justified_messages: current_message_hashes.add(message_hash) while any(current_message_hashes): next_hashes = set() for message_hash in current_message_hashes: message = self.network.global_view.justified_messages[message_hash] messages_needed.add(message) for other_hash in message.justification.values(): if other_hash not in validator.view.pending_messages and \ other_hash not in validator.view.justified_messages: next_hashes.add(other_hash) current_message_hashes = next_hashes return messages_needed def parse(self, protocol_state_string): """Parse the state string!""" for token in protocol_state_string.split(): letter, validator, message = self.parse_token(token) if letter == 'P': self.plot() else: validator = self.validator_set.get_validator_by_name(int(validator)) self.handlers[letter](validator, message) def parse_token(self, token): letter, validator, dash, message, removed_message_names = re.match( self.TOKEN_PATTERN, token ).groups() if letter + validator + dash + message + removed_message_names != token: raise ValueError("Bad token: %s" % token) return letter, validator, message
def create(weight): return ValidatorSet(weight, order_class.View, order_class.Message)