def validate_parent_block_proposer( block: 'Block', parent_block: 'Block', crystallized_state: CrystallizedState, config: Dict[str, Any] = DEFAULT_CONFIG) -> None: if block.slot_number == 0: return proposer_index_in_committee, shard_id = get_proposer_position( parent_block, crystallized_state, config=config, ) if len(block.attestations) == 0: raise Exception("block.attestations should not be an empty list") attestation = block.attestations[0] is_proposer_attestation = (attestation.shard_id == shard_id and attestation.slot == parent_block.slot_number and has_voted(attestation.attester_bitfield, proposer_index_in_committee)) if not is_proposer_attestation: raise Exception( "Proposer of parent block should be one of the attesters in block.attestions[0]:\n" "\tExpected: proposer index in committee: %d, shard_id: %d, slot: %d\n" "\tFound: shard_id: %d, slot: %d, voted: %s" % ( proposer_index_in_committee, shard_id, parent_block.slot_number, attestation.shard_id, attestation.slot, has_voted(attestation.attester_bitfield, proposer_index_in_committee), ))
def validate_attestation(crystallized_state: CrystallizedState, active_state: ActiveState, attestation: 'AttestationRecord', block: 'Block', config: Dict[str, Any]=DEFAULT_CONFIG) -> None: if not attestation.slot < block.slot_number: raise Exception("Attestation slot number too high") if not (attestation.slot > block.slot_number - config['cycle_length']): raise Exception( "Attestation slot number too low:\n" "\tFound: %s, Needed greater than: %s" % (attestation.slot, block.slot_number - config['cycle_length']) ) parent_hashes = get_signed_parent_hashes( active_state, block, attestation, config ) attestation_indices = get_attestation_indices( crystallized_state, attestation, config ) # # validate bitfield # if not (len(attestation.attester_bitfield) == get_bitfield_length(len(attestation_indices))): raise Exception( "Attestation has incorrect bitfield length. Found: %s, Expected: %s" % (len(attestation.attester_bitfield), get_bitfield_length(len(attestation_indices))) ) # check if end bits are zero last_bit = len(attestation_indices) if last_bit % 8 != 0: for i in range(8 - last_bit % 8): if has_voted(attestation.attester_bitfield, last_bit + i): raise Exception("Attestation has non-zero trailing bits") # # validate aggregate_sig # in_cycle_slot_height = attestation.slot % config['cycle_length'] pub_keys = [ crystallized_state.validators[index].pubkey for i, index in enumerate(attestation_indices) if has_voted(attestation.attester_bitfield, i) ] message = blake( in_cycle_slot_height.to_bytes(8, byteorder='big') + b''.join(parent_hashes) + attestation.shard_id.to_bytes(2, byteorder='big') + attestation.shard_block_hash ) if not bls.verify(message, bls.aggregate_pubs(pub_keys), attestation.aggregate_sig): raise Exception("Attestation aggregate signature fails")
def update_ffg_and_crosslink_progress(crystallized_state, crosslinks, ffg_voter_bitfield, votes, config=DEFAULT_CONFIG): # Verify the attestations of crosslink hashes crosslink_votes = { vote.shard_block_hash + vote.shard_id.to_bytes(2, 'big'): vote.voter_bitfield for vote in crosslinks } new_ffg_bitfield = ffg_voter_bitfield total_voters = 0 # The shards that are selected to be crosslinking crosslink_shards = get_crosslink_shards(crystallized_state, config=config) for vote in votes: attestation = get_crosslink_aggvote_msg(vote.shard_id, vote.shard_block_hash, crystallized_state) # Check if this shard is in the crosslink shards list assert vote.shard_id in crosslink_shards indices = get_crosslink_notaries( crystallized_state, vote.shard_id, crosslink_shards=crosslink_shards, config=config, ) votekey = vote.shard_block_hash + vote.shard_id.to_bytes(2, 'big') if votekey not in crosslink_votes: crosslink_votes[votekey] = b"" * get_bitfield_length(len(indices)) bitfield = crosslink_votes[votekey] pubs = [] for i, index in enumerate(indices): if has_voted(vote.notary_bitfield, i): pubs.append(crystallized_state.active_validators[index].pubkey) if has_voted(new_ffg_bitfield, index): new_ffg_bitfield = set_voted(new_ffg_bitfield, index) bitfield = set_voted(bitfield, i) total_voters += 1 assert bls.verify(attestation, bls.aggregate_pubs(pubs), vote.aggregate_sig) crosslink_votes[votekey] = bitfield print('Verified aggregate vote') new_crosslinks = [ PartialCrosslinkRecord(shard_id=int.from_bytes(h[32:], 'big'), shard_block_hash=h[:32], voter_bitfield=crosslink_votes[h]) for h in sorted(crosslink_votes.keys()) ] return new_crosslinks, new_ffg_bitfield, total_voters
def test_bitfield_some_votes(): attesters = list(range(10)) voters = [0, 4, 5, 9] bitfield = get_empty_bitfield(len(attesters)) for voter in voters: bitfield = set_voted(bitfield, voter) assert bitfield == b'\x8c\x40' for attester in attesters: if attester in voters: assert has_voted(bitfield, attester) else: assert not has_voted(bitfield, attester)
def process_ffg_deposits(crystallized_state, ffg_voter_bitfield): total_validators = crystallized_state.num_active_validators finality_distance = crystallized_state.current_epoch - crystallized_state.last_finalized_epoch online_reward = 6 if finality_distance <= 2 else 0 offline_penalty = 3 * finality_distance total_vote_count = 0 total_vote_deposits = 0 deltas = [0] * total_validators for i in range(total_validators): if has_voted(ffg_voter_bitfield, i): total_vote_deposits += crystallized_state.active_validators[ i].balance deltas[i] += online_reward total_vote_count += 1 else: deltas[i] -= offline_penalty print( 'Total voted: %d of %d validators (%.2f%%), %d of %d deposits (%.2f%%)' % (total_vote_count, total_validators, total_vote_count * 100 / total_validators, total_vote_deposits, crystallized_state.total_deposits, total_vote_deposits * 100 / crystallized_state.total_deposits)) print('FFG online reward: %d, offline penalty: %d' % (online_reward, offline_penalty)) print('Total deposit change from FFG: %d' % sum(deltas)) # Check if we need to justify and finalize justify = total_vote_deposits * 3 >= crystallized_state.total_deposits * 2 finalize = False if justify: print('Justifying last epoch') if crystallized_state.last_justified_epoch == crystallized_state.current_epoch - 1: finalize = True print('Finalizing last epoch') return deltas, total_vote_count, total_vote_deposits, justify, finalize
def get_updated_block_vote_cache(crystallized_state, active_state, attestation, block, block_vote_cache, config): new_block_vote_cache = deepcopy(block_vote_cache) parent_hashes = get_signed_parent_hashes(active_state, block, attestation, config) attestation_indices = get_attestation_indices(crystallized_state, attestation, config) for parent_hash in parent_hashes: if parent_hash in attestation.oblique_parent_hashes: continue if parent_hash not in new_block_vote_cache: new_block_vote_cache[parent_hash] = { 'voter_indices': set(), 'total_voter_deposits': 0 } for i, index in enumerate(attestation_indices): if (has_voted(attestation.attester_bitfield, i) and index not in new_block_vote_cache[parent_hash]['voter_indices']): new_block_vote_cache[parent_hash]['voter_indices'].add(index) new_block_vote_cache[parent_hash][ 'total_voter_deposits'] += crystallized_state.validators[ index].balance return new_block_vote_cache
def get_updated_block_vote_cache( crystallized_state: CrystallizedState, active_state: ActiveState, attestation: 'AttestationRecord', block: 'Block', block_vote_cache: BlockVoteCache, config: Dict[str, Any] = DEFAULT_CONFIG) -> BlockVoteCache: new_block_vote_cache = deepcopy(block_vote_cache) parent_hashes = get_signed_parent_hashes(active_state, block, attestation, config) attestation_indices = get_attestation_indices(crystallized_state, attestation, config) for parent_hash in parent_hashes: if parent_hash in attestation.oblique_parent_hashes: continue if parent_hash not in new_block_vote_cache: new_block_vote_cache[parent_hash] = { 'voter_indices': set(), 'total_voter_deposits': 0 } for i, index in enumerate(attestation_indices): if (has_voted(attestation.attester_bitfield, i) and index not in new_block_vote_cache[parent_hash]['voter_indices']): new_block_vote_cache[parent_hash]['voter_indices'].add(index) new_block_vote_cache[parent_hash]['total_voter_deposits'] += ( crystallized_state.validators[index].balance) return new_block_vote_cache
def process_updated_crosslinks(crystallized_state, active_state, config=DEFAULT_CONFIG): total_attestation_balance = {} crosslinks = deepcopy(crystallized_state.crosslink_records) for attestation in active_state.pending_attestations: shard_tuple = (attestation.shard_id, attestation.shard_block_hash) if shard_tuple not in total_attestation_balance: total_attestation_balance[shard_tuple] = 0 attestation_indices = get_attestation_indices(crystallized_state, attestation, config) # find total committee size by balance total_committee_balance = sum([ crystallized_state.validators[index].balance for index in attestation_indices ]) # find votes cast in attestation by balance total_attestation_balance[shard_tuple] += sum([ crystallized_state.validators[index].balance for in_cycle_slot_height, index in enumerate(attestation_indices) if has_voted(attestation.attester_bitfield, in_cycle_slot_height) ]) # if 2/3 of committee voted on crosslink and do no yet have crosslink # for this shard, for this dynasty, add updated crosslink if (3 * total_attestation_balance[shard_tuple] >= 2 * total_committee_balance and crystallized_state.current_dynasty > crosslinks[attestation.shard_id].dynasty): crosslinks[attestation.shard_id] = CrosslinkRecord( dynasty=crystallized_state.current_dynasty, hash=attestation.shard_block_hash) return crosslinks
def test_bitfield_single_votes(): attesters = list(range(10)) bitfield = get_empty_bitfield(len(attesters)) assert set_voted(bitfield, 0) == b'\x80\x00' assert set_voted(bitfield, 1) == b'\x40\x00' assert set_voted(bitfield, 2) == b'\x20\x00' assert set_voted(bitfield, 7) == b'\x01\x00' assert set_voted(bitfield, 8) == b'\x00\x80' assert set_voted(bitfield, 9) == b'\x00\x40' for voter in attesters: bitfield = set_voted(b'\x00\x00', voter) for attester in attesters: if attester == voter: assert has_voted(bitfield, attester) else: assert not has_voted(bitfield, attester)
def test_bitfield_all_votes(): attesters = list(range(10)) bitfield = get_empty_bitfield(len(attesters)) for attester in attesters: bitfield = set_voted(bitfield, attester) for attester in attesters: assert has_voted(bitfield, attester) assert bitfield == b'\xff\xc0'
def process_attestations(validator_set, attestation_indices, attestation_bitfield, msg, aggregate_sig): # Verify the attestations of the parent pubs = [] attesters = [] assert len(attestation_bitfield) == get_bitfield_length( len(attestation_indices)) for i, index in enumerate(attestation_indices): if has_voted(attestation_bitfield, i): pubs.append(validator_set[index].pubkey) attesters.append(index) assert len(attesters) <= 128 assert bls.verify(msg, bls.aggregate_pubs(pubs), aggregate_sig) print('Verified aggregate sig') return attesters
def process_crosslinks(crystallized_state, crosslinks, config=DEFAULT_CONFIG): # Find the most popular crosslink in each shard main_crosslink = {} for c in crosslinks: vote_count = get_vote_count(c.voter_bitfield) if vote_count > main_crosslink.get(c.shard_id, (b'', 0, b''))[1]: main_crosslink[c.shard_id] = (c.shard_block_hash, vote_count, c.voter_bitfield) # Adjust crosslinks new_crosslink_records = [x for x in crystallized_state.crosslink_records] deltas = [0] * crystallized_state.num_active_validators # Process the shards that are selected to be crosslinking... crosslink_shards = get_crosslink_shards(crystallized_state, config=config) for shard in crosslink_shards: indices = get_crosslink_notaries( crystallized_state, shard, crosslink_shards=crosslink_shards, config=config, ) # Get info about the dominant crosslink for this shard h, votes, bitfield = main_crosslink.get( shard, (b'', 0, get_empty_bitfield(len(indices)))) # Calculate rewards for participants and penalties for non-participants crosslink_epoch = crystallized_state.crosslink_records[shard].epoch crosslink_distance = crystallized_state.current_epoch - crosslink_epoch online_reward = 3 if crosslink_distance <= 2 else 0 offline_penalty = crosslink_distance * 2 # Go through participants and evaluate rewards/penalties for i, index in enumerate(indices): if has_voted(bitfield, i): deltas[i] += online_reward else: deltas[i] -= offline_penalty print( 'Shard %d: most recent crosslink %d, reward: (%d, %d), votes: %d of %d (%.2f%%)' % (shard, crystallized_state.crosslink_records[shard].epoch, online_reward, -offline_penalty, votes, len(indices), votes * 100 / len(indices))) # New crosslink if votes * 3 >= len(indices) * 2: new_crosslink_records[shard] = CrosslinkRecord( hash=h, epoch=crystallized_state.current_epoch) print('New crosslink %s' % hex(int.from_bytes(h, 'big'))) print('Total deposit change from crosslinks: %d' % sum(deltas)) return deltas, new_crosslink_records
def test_calculate_crosslink_rewards(genesis_crystallized_state, genesis_active_state, genesis_block, config, mock_make_attestations, mock_make_child): c = genesis_crystallized_state a = genesis_active_state block = genesis_block a.chain = Chain(head=block, blocks=[block]) # progress past first cycle transition # rewards on the following cycle recalc will be based # on what happened during this cycle attestations = mock_make_attestations( (c, a), block, # enough attesters to get a reward but not form a crosslink attester_share=0.58) block2, c2, a2 = mock_make_child( (c, a), block, block.slot_number + config['cycle_length'], attestations) # attestation used for testing attestation = attestations[0] # create a block to trigger next cycle transition attestations2 = mock_make_attestations((c2, a2), block2, attester_share=0.0) block3, c3, a3 = mock_make_child( (c2, a2), block2, block2.slot_number + config['cycle_length'], attestations2) rewards_and_penalties = calculate_crosslink_rewards(c2, a2, block3, config) shard_and_committee = get_shards_and_committees_for_slot( c2, block2.slot_number, config)[0] for committee_index, validator_index in enumerate( shard_and_committee.committee): if has_voted(attestation.attester_bitfield, committee_index): assert rewards_and_penalties[validator_index] > 0 else: assert rewards_and_penalties[validator_index] < 0
def process_updated_crosslinks( crystallized_state: CrystallizedState, active_state: ActiveState, block: 'Block', config: Dict[str, Any] = DEFAULT_CONFIG) -> List[CrosslinkRecord]: total_attestation_balance = {} # type: Dict[Tuple[ShardId, Hash32], int] crosslinks = deepcopy(crystallized_state.crosslink_records) for attestation in active_state.pending_attestations: shard_tuple = (attestation.shard_id, attestation.shard_block_hash) if shard_tuple not in total_attestation_balance: total_attestation_balance[shard_tuple] = 0 attestation_indices = get_attestation_indices(crystallized_state, attestation, config) # find total committee size by balance total_committee_balance = sum([ crystallized_state.validators[index].balance for index in attestation_indices ]) # find votes cast in attestation by balance total_attestation_balance[shard_tuple] += sum([ crystallized_state.validators[index].balance for in_cycle_slot_height, index in enumerate(attestation_indices) if has_voted(attestation.attester_bitfield, in_cycle_slot_height) ]) # if 2/3 of committee voted on crosslink and do no yet have crosslink # for this shard, for this dynasty, add updated crosslink if (3 * total_attestation_balance[shard_tuple] >= 2 * total_committee_balance and crystallized_state.current_dynasty > crosslinks[attestation.shard_id].dynasty): crosslinks[attestation.shard_id] = CrosslinkRecord( dynasty=crystallized_state.current_dynasty, slot=crystallized_state.last_state_recalc + config['cycle_length'], hash=attestation.shard_block_hash) return crosslinks
def test_bitfield_multiple_votes(): bitfield = get_empty_bitfield(1) bitfield = set_voted(bitfield, 0) bitfield = set_voted(bitfield, 0) assert has_voted(bitfield, 0)
def test_empty_bitfield(): attesters = list(range(10)) bitfield = get_empty_bitfield(len(attesters)) for attester in attesters: assert not has_voted(bitfield, attester)
def calculate_crosslink_rewards( crystallized_state: CrystallizedState, active_state: ActiveState, block: 'Block', config: Dict[str, Any] = DEFAULT_CONFIG) -> List[int]: validators = crystallized_state.validators rewards_and_penalties = [0 for _ in validators] # type: List[int] total_deposits = crystallized_state.total_deposits reward_quotient, quadratic_penalty_quotient = get_reward_context( total_deposits, config) last_state_recalc = crystallized_state.last_state_recalc # collect crosslink participation data for each shard_id that was attempted to # be crosslinked two cycles ago committee_crosslinks = {} # type: Dict[str, Any] for slot in range(max(last_state_recalc - config['cycle_length'], 0), last_state_recalc): shards_and_committees = get_shards_and_committees_for_slot( crystallized_state, slot, config=config) for shard_and_committee in shards_and_committees: shard_id = shard_and_committee.shard_id if shard_id not in committee_crosslinks: committee_crosslinks[shard_id] = { 'participating_validator_indices': [], 'non_participating_validator_indices': [], 'total_participated_v_deposits': 0, 'total_v_deposits': 0 } attestations = [ attestation for attestation in active_state.pending_attestations if attestation.slot == slot and attestation.shard_id == shard_id ] if attestations: bitfields = [ attestation.attester_bitfield for attestation in attestations ] bitfield = or_bitfields(bitfields) else: bitfield = get_empty_bitfield( len(shard_and_committee.committee)) committee_crosslink = committee_crosslinks[shard_id] for committee_index, validator_index in enumerate( shard_and_committee.committee): validator = crystallized_state.validators[validator_index] if has_voted(bitfield, committee_index): committee_crosslink[ 'participating_validator_indices'].append( validator_index) committee_crosslink[ 'total_participated_v_deposits'] += validator.balance else: committee_crosslink[ 'non_participating_validator_indices'].append( validator_index) committee_crosslink['total_v_deposits'] += validator.balance # for each shard and associated validator set, apply rewards/penalties based on participation for shard_id, committee_crosslink in committee_crosslinks.items(): crosslink = crystallized_state.crosslink_records[shard_id] if crosslink.dynasty == crystallized_state.current_dynasty: continue time_since_last_confirmation = block.slot_number - crosslink.slot total_participated_v_deposits = committee_crosslink[ 'total_participated_v_deposits'] total_v_deposits = committee_crosslink['total_v_deposits'] for validator_index in committee_crosslink[ 'participating_validator_indices']: validator = crystallized_state.validators[validator_index] rewards_and_penalties[validator_index] += ( validator.balance // reward_quotient * (2 * total_participated_v_deposits - total_v_deposits) // total_v_deposits) for validator_index in committee_crosslink[ 'non_participating_validator_indices']: validator = crystallized_state.validators[validator_index] rewards_and_penalties[validator_index] -= ( (validator.balance // reward_quotient) + (validator.balance * time_since_last_confirmation // quadratic_penalty_quotient)) return rewards_and_penalties
def validate_attestation(crystallized_state: CrystallizedState, active_state: ActiveState, attestation: 'AttestationRecord', block: 'Block', parent_block: 'Block', config: Dict[str, Any] = DEFAULT_CONFIG) -> None: # Verify attestation.slot_number if not attestation.slot <= parent_block.slot_number: raise Exception("Attestation slot number too high:\n" "\tFound: %s Needed less than or equal to %s" % (attestation.slot, parent_block.slot_number)) if not (attestation.slot >= max( parent_block.slot_number - config['cycle_length'] + 1, 0)): raise Exception( "Attestation slot number too low:\n" "\tFound: %s, Needed greater than or equalt to: %s" % (attestation.slot, max(parent_block.slot_number - config['cycle_length'] + 1, 0))) # TODO: Verify that the justified_slot and justified_block_hash given are in # the chain and are equal to or earlier than the last_justified_slot # in the crystallized state. parent_hashes = get_signed_parent_hashes(active_state, block, attestation, config) attestation_indices = get_attestation_indices(crystallized_state, attestation, config) # # validate bitfield # if not (len(attestation.attester_bitfield) == get_bitfield_length( len(attestation_indices))): raise Exception( "Attestation has incorrect bitfield length. Found: %s, Expected: %s" % (len(attestation.attester_bitfield), get_bitfield_length(len(attestation_indices)))) # check if end bits are zero last_bit = len(attestation_indices) if last_bit % 8 != 0: for i in range(8 - last_bit % 8): if has_voted(attestation.attester_bitfield, last_bit + i): raise Exception("Attestation has non-zero trailing bits") # # validate aggregate_sig # pub_keys = [ crystallized_state.validators[index].pubkey for i, index in enumerate(attestation_indices) if has_voted(attestation.attester_bitfield, i) ] message = blake( attestation.slot.to_bytes(8, byteorder='big') + b''.join(parent_hashes) + attestation.shard_id.to_bytes(2, byteorder='big') + attestation.shard_block_hash + attestation.justified_slot.to_bytes(8, 'big')) if not bls.verify(message, bls.aggregate_pubs(pub_keys), attestation.aggregate_sig): raise Exception("Attestation aggregate signature fails")
def validate_attestation(crystallized_state: CrystallizedState, active_state: ActiveState, attestation: 'AttestationRecord', block: 'Block', parent_block: 'Block', config: Dict[str, Any] = DEFAULT_CONFIG) -> bool: # # validate slot number # if not attestation.slot <= parent_block.slot_number: raise ValidationError("Attestation slot number too high:\n" "\tFound: %s Needed less than or equal to %s" % (attestation.slot, parent_block.slot_number)) if not (attestation.slot >= max( parent_block.slot_number - config['cycle_length'] + 1, 0)): raise ValidationError( "Attestation slot number too low:\n" "\tFound: %s, Needed greater than or equalt to: %s" % (attestation.slot, max(parent_block.slot_number - config['cycle_length'] + 1, 0))) # # validate justified_slot and justified_block_hash # if attestation.justified_slot > crystallized_state.last_justified_slot: raise ValidationError( "attestation.justified_slot %s should be equal to or earlier than" " crystallized_state.last_justified_slot %s" % ( attestation.justified_slot, crystallized_state.last_justified_slot, )) justified_block = active_state.chain.get_block_by_hash( attestation.justified_block_hash) if justified_block is None: raise ValidationError( "justified_block_hash %s is not in the canonical chain" % attestation.justified_block_hash) if justified_block.slot_number != attestation.justified_slot: raise ValidationError( "justified_slot %s doesn't match justified_block_hash" % attestation.justified_slot) parent_hashes = get_signed_parent_hashes(active_state, block, attestation, config) attestation_indices = get_attestation_indices(crystallized_state, attestation, config) # # validate bitfield # if not (len(attestation.attester_bitfield) == get_bitfield_length( len(attestation_indices))): raise ValidationError( "Attestation has incorrect bitfield length. Found: %s, Expected: %s" % (len(attestation.attester_bitfield), get_bitfield_length(len(attestation_indices)))) # check if end bits are zero last_bit = len(attestation_indices) if last_bit % 8 != 0: for i in range(8 - last_bit % 8): if has_voted(attestation.attester_bitfield, last_bit + i): raise ValidationError("Attestation has non-zero trailing bits") # # validate aggregate_sig # pub_keys = [ crystallized_state.validators[validator_index].pubkey for committee_index, validator_index in enumerate(attestation_indices) if has_voted(attestation.attester_bitfield, committee_index) ] message = blake( attestation.slot.to_bytes(8, byteorder='big') + b''.join(parent_hashes) + attestation.shard_id.to_bytes(2, byteorder='big') + attestation.shard_block_hash + attestation.justified_slot.to_bytes(8, 'big')) if not bls.verify(message, bls.aggregate_pubs(pub_keys), attestation.aggregate_sig): raise ValidationError("Attestation aggregate signature fails") return True