def mock_make_attestations(parent_state, block, attester_share=0.8): crystallized_state, active_state = parent_state cycle_length = config['cycle_length'] in_cycle_slot_height = block.slot_number % cycle_length indices = crystallized_state.indices_for_slots[cycle_length + in_cycle_slot_height] print("Generating attestations for shards: %s" % len(indices)) attestations = [] for shard_and_committee in indices: shard_id = shard_and_committee.shard_id committee_indices = shard_and_committee.committee print("Generating attestation for shard %s" % shard_id) print("Committee size %s" % len(committee_indices)) # Create attestation attestation = AttestationRecord( slot=block.slot_number, shard_id=shard_and_committee.shard_id, oblique_parent_hashes=[], shard_block_hash=blake(bytes(str(shard_id), 'utf-8')), attester_bitfield=get_empty_bitfield(len(committee_indices))) # Randomly pick indices to include is_attesting = [ random.random() < attester_share for _ in range(len(committee_indices)) ] # Proposer always attests is_attesting[0] = True # Generating signatures and aggregating result parent_hashes = get_hashes_to_sign(active_state, block, config) message = blake( attestation.slot.to_bytes(8, byteorder='big') + b''.join(parent_hashes) + shard_id.to_bytes(2, byteorder='big') + attestation.shard_block_hash) sigs = [ bls.sign(message, keymap[crystallized_state.validators[indice].pubkey]) for i, indice in enumerate(committee_indices) if is_attesting[i] ] attestation.aggregate_sig = bls.aggregate_sigs(sigs) print('Aggregated sig') attestation_bitfield = get_empty_bitfield(len(committee_indices)) for i, attesting in enumerate(is_attesting): if attesting: attestation_bitfield = set_voted(attestation_bitfield, i) attestation.attester_bitfield = attestation_bitfield print('Aggregate bitfield:', bin(int.from_bytes(attestation_bitfield, 'big'))) attestations.append(attestation) return attestations
def test_validate_attestation_attester_bitfield(attestation_validation_fixture, config): (crystallized_state, active_state, original_attestation, block, parent_block) = attestation_validation_fixture attestation = copy.deepcopy(original_attestation) attestation.attester_bitfield = get_empty_bitfield(10) with pytest.raises(ValidationError): validate_attestation( crystallized_state, active_state, attestation, block, parent_block, config, ) attestation = copy.deepcopy(original_attestation) attestation_indices = get_attestation_indices(crystallized_state, attestation, config) last_bit = len(attestation_indices) attestation.attester_bitfield = set_voted(attestation.attester_bitfield, last_bit) with pytest.raises(ValidationError): validate_attestation( crystallized_state, active_state, attestation, block, parent_block, config, )
def genesis_active_state(genesis_crystallized_state, init_randao): return ActiveState(height=1, randao=init_randao, ffg_voter_bitfield=get_empty_bitfield( genesis_crystallized_state.num_active_validators), balance_deltas=[], partial_crosslinks=[], total_skip_count=0)
def test_bitfield_all_votes(): attesters = list(range(10)) bitfield = get_empty_bitfield(len(attesters)) for attester in attesters: bitfield = set_voted(bitfield, attester) for attester in attesters: assert has_voted(bitfield, attester) assert bitfield == b'\xff\xc0'
def test_bitfield_some_votes(): attesters = list(range(10)) voters = [0, 4, 5, 9] bitfield = get_empty_bitfield(len(attesters)) for voter in voters: bitfield = set_voted(bitfield, voter) assert bitfield == b'\x8c\x40' for attester in attesters: if attester in voters: assert has_voted(bitfield, attester) else: assert not has_voted(bitfield, attester)
def process_crosslinks(crystallized_state, crosslinks, config=DEFAULT_CONFIG): # Find the most popular crosslink in each shard main_crosslink = {} for c in crosslinks: vote_count = get_vote_count(c.voter_bitfield) if vote_count > main_crosslink.get(c.shard_id, (b'', 0, b''))[1]: main_crosslink[c.shard_id] = (c.shard_block_hash, vote_count, c.voter_bitfield) # Adjust crosslinks new_crosslink_records = [x for x in crystallized_state.crosslink_records] deltas = [0] * crystallized_state.num_active_validators # Process the shards that are selected to be crosslinking... crosslink_shards = get_crosslink_shards(crystallized_state, config=config) for shard in crosslink_shards: indices = get_crosslink_notaries( crystallized_state, shard, crosslink_shards=crosslink_shards, config=config, ) # Get info about the dominant crosslink for this shard h, votes, bitfield = main_crosslink.get( shard, (b'', 0, get_empty_bitfield(len(indices)))) # Calculate rewards for participants and penalties for non-participants crosslink_epoch = crystallized_state.crosslink_records[shard].epoch crosslink_distance = crystallized_state.current_epoch - crosslink_epoch online_reward = 3 if crosslink_distance <= 2 else 0 offline_penalty = crosslink_distance * 2 # Go through participants and evaluate rewards/penalties for i, index in enumerate(indices): if has_voted(bitfield, i): deltas[i] += online_reward else: deltas[i] -= offline_penalty print( 'Shard %d: most recent crosslink %d, reward: (%d, %d), votes: %d of %d (%.2f%%)' % (shard, crystallized_state.crosslink_records[shard].epoch, online_reward, -offline_penalty, votes, len(indices), votes * 100 / len(indices))) # New crosslink if votes * 3 >= len(indices) * 2: new_crosslink_records[shard] = CrosslinkRecord( hash=h, epoch=crystallized_state.current_epoch) print('New crosslink %s' % hex(int.from_bytes(h, 'big'))) print('Total deposit change from crosslinks: %d' % sum(deltas)) return deltas, new_crosslink_records
def test_bitfield_single_votes(): attesters = list(range(10)) bitfield = get_empty_bitfield(len(attesters)) assert set_voted(bitfield, 0) == b'\x80\x00' assert set_voted(bitfield, 1) == b'\x40\x00' assert set_voted(bitfield, 2) == b'\x20\x00' assert set_voted(bitfield, 7) == b'\x01\x00' assert set_voted(bitfield, 8) == b'\x00\x80' assert set_voted(bitfield, 9) == b'\x00\x40' for voter in attesters: bitfield = set_voted(b'\x00\x00', voter) for attester in attesters: if attester == voter: assert has_voted(bitfield, attester) else: assert not has_voted(bitfield, attester)
def mock_make_attestations(parent_state, block, attester_share=0.8): crystallized_state, active_state = parent_state cycle_length = config['cycle_length'] in_cycle_slot_height = block.slot_number % cycle_length indices = crystallized_state.shard_and_committee_for_slots[ cycle_length + in_cycle_slot_height] print("Generating attestations for shards: %s" % len(indices)) proposer_index_in_committee, proposer_shard_id = get_proposer_position( block, crystallized_state, config=config, ) attestations = [] for shard_and_committee in indices: shard_id = shard_and_committee.shard_id committee_indices = shard_and_committee.committee print("Generating attestation for shard %s" % shard_id) print("Committee size %s" % len(committee_indices)) justified_slot = crystallized_state.last_justified_slot justified_block_hash = active_state.chain.get_block_by_slot_number( justified_slot).hash # Create attestation attestation = AttestationRecord( slot=block.slot_number, shard_id=shard_and_committee.shard_id, oblique_parent_hashes=[], shard_block_hash=blake(bytes(str(shard_id), 'utf-8')), attester_bitfield=get_empty_bitfield(len(committee_indices)), justified_slot=justified_slot, justified_block_hash=justified_block_hash, ) # fill with roughly attester share fraction of voters is_attesting = [ i < attester_share * len(committee_indices) for i in range(len(committee_indices)) ] # Proposer always attests if shard_id == proposer_shard_id: is_attesting[proposer_index_in_committee] = True # Generating signatures and aggregating result parent_hashes = get_hashes_to_sign(active_state, block, config) message = blake( attestation.slot.to_bytes(8, byteorder='big') + b''.join(parent_hashes) + shard_id.to_bytes(2, byteorder='big') + attestation.shard_block_hash + attestation.justified_slot.to_bytes(8, byteorder='big')) sigs = [ bls.sign(message, keymap[crystallized_state.validators[indice].pubkey]) for i, indice in enumerate(committee_indices) if is_attesting[i] ] attestation.aggregate_sig = bls.aggregate_sigs(sigs) print('Aggregated sig') attestation_bitfield = get_empty_bitfield(len(committee_indices)) for i, attesting in enumerate(is_attesting): if attesting: attestation_bitfield = set_voted(attestation_bitfield, i) attestation.attester_bitfield = attestation_bitfield print('Aggregate bitfield:', bin(int.from_bytes(attestation_bitfield, 'big'))) attestations.append(attestation) return attestations
def make_unfinished_block(parent_state, parent, skips, attester_share=0.8, crosslink_shards_and_shares=None): if crosslink_shards_and_shares is None: crosslink_shards_and_shares = [] crystallized_state, active_state = parent_state parent_attestation = serialize(parent) indices, proposer = get_attesters_and_proposer( crystallized_state, active_state, skips, config ) print('Selected indices: %r' % indices) print('Selected block proposer: %d' % proposer) # Randomly pick indices to include is_attesting = [random.random() < attester_share for _ in indices] # Attestations sigs = [ bls.sign( parent_attestation, keymap[crystallized_state.active_validators[indices[i]].pubkey] ) for i, attesting in enumerate(is_attesting) if attesting ] attestation_aggregate_sig = bls.aggregate_sigs(sigs) print('Aggregated sig') attestation_bitfield = get_empty_bitfield(len(indices)) for i, attesting in enumerate(is_attesting): if attesting: attestation_bitfield = set_voted(attestation_bitfield, i) print('Aggregate bitfield:', bin(int.from_bytes(attestation_bitfield, 'big'))) # Randomly pick indices to include for crosslinks shard_aggregate_votes = [] # The shards that are selected to be crosslinking crosslink_shards = get_crosslink_shards(crystallized_state, config=config) for shard, crosslinker_share in crosslink_shards_and_shares: # Check if this shard is in the crosslink shards list assert shard in crosslink_shards print('Making crosslink in shard %d' % shard) indices = get_crosslink_notaries(crystallized_state, shard, crosslink_shards=crosslink_shards, config=config) print('Indices: %r' % indices) is_notarizing = [random.random() < attester_share for _ in indices] notary_bitfield = get_empty_bitfield(len(indices)) for i, notarizing in enumerate(is_notarizing): if notarizing: notary_bitfield = set_voted(notary_bitfield, i) print('Bitfield:', bin(int.from_bytes(notary_bitfield, 'big'))) shard_block_hash = blake(bytes([shard])) crosslink_attestation_hash = get_crosslink_aggvote_msg( shard, shard_block_hash, crystallized_state ) sigs = [ bls.sign( crosslink_attestation_hash, keymap[crystallized_state.active_validators[indices[i]].pubkey] ) for i, notarizing in enumerate(is_notarizing) if notarizing ] v = AggregateVote( shard_id=shard, shard_block_hash=shard_block_hash, notary_bitfield=notary_bitfield, aggregate_sig=list(bls.aggregate_sigs(sigs)) ) shard_aggregate_votes.append(v) print('Added %d shard aggregate votes' % len(crosslink_shards_and_shares)) block = Block( parent_hash=blake(parent_attestation), skip_count=skips, randao_reveal=blake(str(random.random()).encode('utf-8')), attestation_bitfield=attestation_bitfield, attestation_aggregate_sig=list(attestation_aggregate_sig), shard_aggregate_votes=shard_aggregate_votes, main_chain_ref=b'\x00'*32, state_hash=b'\x00'*64 ) return block, proposer
def _initialize_new_epoch(crystallized_state, active_state, config=DEFAULT_CONFIG): print('Processing epoch transition') # Process rewards from FFG/crosslink votes new_validator_records = deepcopy(crystallized_state.active_validators) # Who voted in the last epoch ffg_voter_bitfield = active_state.ffg_voter_bitfield # Balance changes, and total vote counts for FFG deltas_casper, total_vote_count, total_vote_deposits, justify, finalize = \ process_ffg_deposits(crystallized_state, ffg_voter_bitfield) # Balance changes, and total vote counts for crosslinks deltas_crosslinks, new_crosslink_records = process_crosslinks( crystallized_state, active_state.partial_crosslinks, config=config, ) # process recent attesters balance deltas deltas_recent_attesters = process_recent_attesters( crystallized_state, active_state.recent_attesters, config=config, ) # process recent proposers balance deltas deltas_recent_proposers = process_recent_proposers( crystallized_state, active_state.recent_proposers) for i, validator in enumerate(new_validator_records): validator.balance += (deltas_casper[i] + deltas_crosslinks[i] + deltas_recent_attesters[i] + deltas_recent_proposers[i]) total_deposits = crystallized_state.total_deposits + sum( deltas_casper + deltas_crosslinks + deltas_recent_attesters + deltas_recent_proposers) print('New total deposits: %d' % total_deposits) if justify: last_justified_epoch = crystallized_state.current_epoch else: last_justified_epoch = crystallized_state.last_justified_epoch if finalize: last_finalized_epoch = crystallized_state.current_epoch - 1 dynasty = crystallized_state.dynasty + 1 new_queued_validators, new_active_validators, new_exited_validators = \ get_incremented_validator_sets(crystallized_state, new_validator_records) else: last_finalized_epoch = crystallized_state.last_finalized_epoch dynasty = crystallized_state.dynasty new_queued_validators = crystallized_state.queued_validators new_active_validators = crystallized_state.active_validators new_exited_validators = crystallized_state.exited_validators crystallized_state = CrystallizedState( queued_validators=new_queued_validators, active_validators=new_active_validators, exited_validators=new_exited_validators, current_shuffling=get_shuffling(active_state.randao, len(new_active_validators), config=config), last_justified_epoch=last_justified_epoch, last_finalized_epoch=last_finalized_epoch, dynasty=dynasty, next_shard=0, current_epoch=crystallized_state.current_epoch + 1, crosslink_records=new_crosslink_records, total_deposits=total_deposits) # Reset the active state active_state = ActiveState(height=active_state.height, randao=active_state.randao, ffg_voter_bitfield=get_empty_bitfield( crystallized_state.num_active_validators), balance_deltas=[], partial_crosslinks=[], total_skip_count=active_state.total_skip_count, recent_proposers=[]) return crystallized_state, active_state
def test_bitfield_multiple_votes(): bitfield = get_empty_bitfield(1) bitfield = set_voted(bitfield, 0) bitfield = set_voted(bitfield, 0) assert has_voted(bitfield, 0)
def test_empty_bitfield(): attesters = list(range(10)) bitfield = get_empty_bitfield(len(attesters)) for attester in attesters: assert not has_voted(bitfield, attester)
def calculate_crosslink_rewards( crystallized_state: CrystallizedState, active_state: ActiveState, block: 'Block', config: Dict[str, Any] = DEFAULT_CONFIG) -> List[int]: validators = crystallized_state.validators rewards_and_penalties = [0 for _ in validators] # type: List[int] total_deposits = crystallized_state.total_deposits reward_quotient, quadratic_penalty_quotient = get_reward_context( total_deposits, config) last_state_recalc = crystallized_state.last_state_recalc # collect crosslink participation data for each shard_id that was attempted to # be crosslinked two cycles ago committee_crosslinks = {} # type: Dict[str, Any] for slot in range(max(last_state_recalc - config['cycle_length'], 0), last_state_recalc): shards_and_committees = get_shards_and_committees_for_slot( crystallized_state, slot, config=config) for shard_and_committee in shards_and_committees: shard_id = shard_and_committee.shard_id if shard_id not in committee_crosslinks: committee_crosslinks[shard_id] = { 'participating_validator_indices': [], 'non_participating_validator_indices': [], 'total_participated_v_deposits': 0, 'total_v_deposits': 0 } attestations = [ attestation for attestation in active_state.pending_attestations if attestation.slot == slot and attestation.shard_id == shard_id ] if attestations: bitfields = [ attestation.attester_bitfield for attestation in attestations ] bitfield = or_bitfields(bitfields) else: bitfield = get_empty_bitfield( len(shard_and_committee.committee)) committee_crosslink = committee_crosslinks[shard_id] for committee_index, validator_index in enumerate( shard_and_committee.committee): validator = crystallized_state.validators[validator_index] if has_voted(bitfield, committee_index): committee_crosslink[ 'participating_validator_indices'].append( validator_index) committee_crosslink[ 'total_participated_v_deposits'] += validator.balance else: committee_crosslink[ 'non_participating_validator_indices'].append( validator_index) committee_crosslink['total_v_deposits'] += validator.balance # for each shard and associated validator set, apply rewards/penalties based on participation for shard_id, committee_crosslink in committee_crosslinks.items(): crosslink = crystallized_state.crosslink_records[shard_id] if crosslink.dynasty == crystallized_state.current_dynasty: continue time_since_last_confirmation = block.slot_number - crosslink.slot total_participated_v_deposits = committee_crosslink[ 'total_participated_v_deposits'] total_v_deposits = committee_crosslink['total_v_deposits'] for validator_index in committee_crosslink[ 'participating_validator_indices']: validator = crystallized_state.validators[validator_index] rewards_and_penalties[validator_index] += ( validator.balance // reward_quotient * (2 * total_participated_v_deposits - total_v_deposits) // total_v_deposits) for validator_index in committee_crosslink[ 'non_participating_validator_indices']: validator = crystallized_state.validators[validator_index] rewards_and_penalties[validator_index] -= ( (validator.balance // reward_quotient) + (validator.balance * time_since_last_confirmation // quadratic_penalty_quotient)) return rewards_and_penalties