def mock_make_attestations(parent_state, block, attester_share=0.8):
crystallized_state, active_state = parent_state
cycle_length = config['cycle_length']
in_cycle_slot_height = block.slot_number % cycle_length
indices = crystallized_state.indices_for_slots[cycle_length +
in_cycle_slot_height]
print("Generating attestations for shards: %s" % len(indices))
attestations = []
for shard_and_committee in indices:
shard_id = shard_and_committee.shard_id
committee_indices = shard_and_committee.committee
print("Generating attestation for shard %s" % shard_id)
print("Committee size %s" % len(committee_indices))
# Create attestation
attestation = AttestationRecord(
slot=block.slot_number,
shard_id=shard_and_committee.shard_id,
oblique_parent_hashes=[],
shard_block_hash=blake(bytes(str(shard_id), 'utf-8')),
attester_bitfield=get_empty_bitfield(len(committee_indices)))
# Randomly pick indices to include
is_attesting = [
random.random() < attester_share
for _ in range(len(committee_indices))
]
# Proposer always attests
is_attesting[0] = True
# Generating signatures and aggregating result
parent_hashes = get_hashes_to_sign(active_state, block, config)
message = blake(
attestation.slot.to_bytes(8, byteorder='big') +
b''.join(parent_hashes) +
shard_id.to_bytes(2, byteorder='big') +
attestation.shard_block_hash)
sigs = [
bls.sign(message,
keymap[crystallized_state.validators[indice].pubkey])
for i, indice in enumerate(committee_indices)
if is_attesting[i]
]
attestation.aggregate_sig = bls.aggregate_sigs(sigs)
print('Aggregated sig')
attestation_bitfield = get_empty_bitfield(len(committee_indices))
for i, attesting in enumerate(is_attesting):
if attesting:
attestation_bitfield = set_voted(attestation_bitfield, i)
attestation.attester_bitfield = attestation_bitfield
print('Aggregate bitfield:',
bin(int.from_bytes(attestation_bitfield, 'big')))
attestations.append(attestation)
return attestations
def test_validate_attestation_attester_bitfield(attestation_validation_fixture,
config):
(crystallized_state, active_state, original_attestation, block,
parent_block) = attestation_validation_fixture
attestation = copy.deepcopy(original_attestation)
attestation.attester_bitfield = get_empty_bitfield(10)
with pytest.raises(ValidationError):
validate_attestation(
crystallized_state,
active_state,
attestation,
block,
parent_block,
config,
)
attestation = copy.deepcopy(original_attestation)
attestation_indices = get_attestation_indices(crystallized_state,
attestation, config)
last_bit = len(attestation_indices)
attestation.attester_bitfield = set_voted(attestation.attester_bitfield,
last_bit)
with pytest.raises(ValidationError):
validate_attestation(
crystallized_state,
active_state,
attestation,
block,
parent_block,
config,
)
def genesis_active_state(genesis_crystallized_state, init_randao):
return ActiveState(height=1,
randao=init_randao,
ffg_voter_bitfield=get_empty_bitfield(
genesis_crystallized_state.num_active_validators),
balance_deltas=[],
partial_crosslinks=[],
total_skip_count=0)
def test_bitfield_all_votes():
attesters = list(range(10))
bitfield = get_empty_bitfield(len(attesters))
for attester in attesters:
bitfield = set_voted(bitfield, attester)
for attester in attesters:
assert has_voted(bitfield, attester)
assert bitfield == b'\xff\xc0'
def test_bitfield_some_votes():
attesters = list(range(10))
voters = [0, 4, 5, 9]
bitfield = get_empty_bitfield(len(attesters))
for voter in voters:
bitfield = set_voted(bitfield, voter)
assert bitfield == b'\x8c\x40'
for attester in attesters:
if attester in voters:
assert has_voted(bitfield, attester)
else:
assert not has_voted(bitfield, attester)
def process_crosslinks(crystallized_state, crosslinks, config=DEFAULT_CONFIG):
# Find the most popular crosslink in each shard
main_crosslink = {}
for c in crosslinks:
vote_count = get_vote_count(c.voter_bitfield)
if vote_count > main_crosslink.get(c.shard_id, (b'', 0, b''))[1]:
main_crosslink[c.shard_id] = (c.shard_block_hash, vote_count,
c.voter_bitfield)
# Adjust crosslinks
new_crosslink_records = [x for x in crystallized_state.crosslink_records]
deltas = [0] * crystallized_state.num_active_validators
# Process the shards that are selected to be crosslinking...
crosslink_shards = get_crosslink_shards(crystallized_state, config=config)
for shard in crosslink_shards:
indices = get_crosslink_notaries(
crystallized_state,
shard,
crosslink_shards=crosslink_shards,
config=config,
)
# Get info about the dominant crosslink for this shard
h, votes, bitfield = main_crosslink.get(
shard, (b'', 0, get_empty_bitfield(len(indices))))
# Calculate rewards for participants and penalties for non-participants
crosslink_epoch = crystallized_state.crosslink_records[shard].epoch
crosslink_distance = crystallized_state.current_epoch - crosslink_epoch
online_reward = 3 if crosslink_distance <= 2 else 0
offline_penalty = crosslink_distance * 2
# Go through participants and evaluate rewards/penalties
for i, index in enumerate(indices):
if has_voted(bitfield, i):
deltas[i] += online_reward
else:
deltas[i] -= offline_penalty
print(
'Shard %d: most recent crosslink %d, reward: (%d, %d), votes: %d of %d (%.2f%%)'
% (shard, crystallized_state.crosslink_records[shard].epoch,
online_reward, -offline_penalty, votes, len(indices),
votes * 100 / len(indices)))
# New crosslink
if votes * 3 >= len(indices) * 2:
new_crosslink_records[shard] = CrosslinkRecord(
hash=h, epoch=crystallized_state.current_epoch)
print('New crosslink %s' % hex(int.from_bytes(h, 'big')))
print('Total deposit change from crosslinks: %d' % sum(deltas))
return deltas, new_crosslink_records
def test_bitfield_single_votes():
attesters = list(range(10))
bitfield = get_empty_bitfield(len(attesters))
assert set_voted(bitfield, 0) == b'\x80\x00'
assert set_voted(bitfield, 1) == b'\x40\x00'
assert set_voted(bitfield, 2) == b'\x20\x00'
assert set_voted(bitfield, 7) == b'\x01\x00'
assert set_voted(bitfield, 8) == b'\x00\x80'
assert set_voted(bitfield, 9) == b'\x00\x40'
for voter in attesters:
bitfield = set_voted(b'\x00\x00', voter)
for attester in attesters:
if attester == voter:
assert has_voted(bitfield, attester)
else:
assert not has_voted(bitfield, attester)
def mock_make_attestations(parent_state, block, attester_share=0.8):
crystallized_state, active_state = parent_state
cycle_length = config['cycle_length']
in_cycle_slot_height = block.slot_number % cycle_length
indices = crystallized_state.shard_and_committee_for_slots[
cycle_length + in_cycle_slot_height]
print("Generating attestations for shards: %s" % len(indices))
proposer_index_in_committee, proposer_shard_id = get_proposer_position(
block,
crystallized_state,
config=config,
)
attestations = []
for shard_and_committee in indices:
shard_id = shard_and_committee.shard_id
committee_indices = shard_and_committee.committee
print("Generating attestation for shard %s" % shard_id)
print("Committee size %s" % len(committee_indices))
justified_slot = crystallized_state.last_justified_slot
justified_block_hash = active_state.chain.get_block_by_slot_number(
justified_slot).hash
# Create attestation
attestation = AttestationRecord(
slot=block.slot_number,
shard_id=shard_and_committee.shard_id,
oblique_parent_hashes=[],
shard_block_hash=blake(bytes(str(shard_id), 'utf-8')),
attester_bitfield=get_empty_bitfield(len(committee_indices)),
justified_slot=justified_slot,
justified_block_hash=justified_block_hash,
)
# fill with roughly attester share fraction of voters
is_attesting = [
i < attester_share * len(committee_indices)
for i in range(len(committee_indices))
]
# Proposer always attests
if shard_id == proposer_shard_id:
is_attesting[proposer_index_in_committee] = True
# Generating signatures and aggregating result
parent_hashes = get_hashes_to_sign(active_state, block, config)
message = blake(
attestation.slot.to_bytes(8, byteorder='big') +
b''.join(parent_hashes) +
shard_id.to_bytes(2, byteorder='big') +
attestation.shard_block_hash +
attestation.justified_slot.to_bytes(8, byteorder='big'))
sigs = [
bls.sign(message,
keymap[crystallized_state.validators[indice].pubkey])
for i, indice in enumerate(committee_indices)
if is_attesting[i]
]
attestation.aggregate_sig = bls.aggregate_sigs(sigs)
print('Aggregated sig')
attestation_bitfield = get_empty_bitfield(len(committee_indices))
for i, attesting in enumerate(is_attesting):
if attesting:
attestation_bitfield = set_voted(attestation_bitfield, i)
attestation.attester_bitfield = attestation_bitfield
print('Aggregate bitfield:',
bin(int.from_bytes(attestation_bitfield, 'big')))
attestations.append(attestation)
return attestations
def make_unfinished_block(parent_state,
parent,
skips,
attester_share=0.8,
crosslink_shards_and_shares=None):
if crosslink_shards_and_shares is None:
crosslink_shards_and_shares = []
crystallized_state, active_state = parent_state
parent_attestation = serialize(parent)
indices, proposer = get_attesters_and_proposer(
crystallized_state,
active_state,
skips,
config
)
print('Selected indices: %r' % indices)
print('Selected block proposer: %d' % proposer)
# Randomly pick indices to include
is_attesting = [random.random() < attester_share for _ in indices]
# Attestations
sigs = [
bls.sign(
parent_attestation,
keymap[crystallized_state.active_validators[indices[i]].pubkey]
)
for i, attesting in enumerate(is_attesting) if attesting
]
attestation_aggregate_sig = bls.aggregate_sigs(sigs)
print('Aggregated sig')
attestation_bitfield = get_empty_bitfield(len(indices))
for i, attesting in enumerate(is_attesting):
if attesting:
attestation_bitfield = set_voted(attestation_bitfield, i)
print('Aggregate bitfield:', bin(int.from_bytes(attestation_bitfield, 'big')))
# Randomly pick indices to include for crosslinks
shard_aggregate_votes = []
# The shards that are selected to be crosslinking
crosslink_shards = get_crosslink_shards(crystallized_state, config=config)
for shard, crosslinker_share in crosslink_shards_and_shares:
# Check if this shard is in the crosslink shards list
assert shard in crosslink_shards
print('Making crosslink in shard %d' % shard)
indices = get_crosslink_notaries(crystallized_state, shard, crosslink_shards=crosslink_shards, config=config)
print('Indices: %r' % indices)
is_notarizing = [random.random() < attester_share for _ in indices]
notary_bitfield = get_empty_bitfield(len(indices))
for i, notarizing in enumerate(is_notarizing):
if notarizing:
notary_bitfield = set_voted(notary_bitfield, i)
print('Bitfield:', bin(int.from_bytes(notary_bitfield, 'big')))
shard_block_hash = blake(bytes([shard]))
crosslink_attestation_hash = get_crosslink_aggvote_msg(
shard,
shard_block_hash,
crystallized_state
)
sigs = [
bls.sign(
crosslink_attestation_hash,
keymap[crystallized_state.active_validators[indices[i]].pubkey]
)
for i, notarizing in enumerate(is_notarizing) if notarizing
]
v = AggregateVote(
shard_id=shard,
shard_block_hash=shard_block_hash,
notary_bitfield=notary_bitfield,
aggregate_sig=list(bls.aggregate_sigs(sigs))
)
shard_aggregate_votes.append(v)
print('Added %d shard aggregate votes' % len(crosslink_shards_and_shares))
block = Block(
parent_hash=blake(parent_attestation),
skip_count=skips,
randao_reveal=blake(str(random.random()).encode('utf-8')),
attestation_bitfield=attestation_bitfield,
attestation_aggregate_sig=list(attestation_aggregate_sig),
shard_aggregate_votes=shard_aggregate_votes,
main_chain_ref=b'\x00'*32,
state_hash=b'\x00'*64
)
return block, proposer
def _initialize_new_epoch(crystallized_state,
active_state,
config=DEFAULT_CONFIG):
print('Processing epoch transition')
# Process rewards from FFG/crosslink votes
new_validator_records = deepcopy(crystallized_state.active_validators)
# Who voted in the last epoch
ffg_voter_bitfield = active_state.ffg_voter_bitfield
# Balance changes, and total vote counts for FFG
deltas_casper, total_vote_count, total_vote_deposits, justify, finalize = \
process_ffg_deposits(crystallized_state, ffg_voter_bitfield)
# Balance changes, and total vote counts for crosslinks
deltas_crosslinks, new_crosslink_records = process_crosslinks(
crystallized_state,
active_state.partial_crosslinks,
config=config,
)
# process recent attesters balance deltas
deltas_recent_attesters = process_recent_attesters(
crystallized_state,
active_state.recent_attesters,
config=config,
)
# process recent proposers balance deltas
deltas_recent_proposers = process_recent_proposers(
crystallized_state, active_state.recent_proposers)
for i, validator in enumerate(new_validator_records):
validator.balance += (deltas_casper[i] + deltas_crosslinks[i] +
deltas_recent_attesters[i] +
deltas_recent_proposers[i])
total_deposits = crystallized_state.total_deposits + sum(
deltas_casper + deltas_crosslinks + deltas_recent_attesters +
deltas_recent_proposers)
print('New total deposits: %d' % total_deposits)
if justify:
last_justified_epoch = crystallized_state.current_epoch
else:
last_justified_epoch = crystallized_state.last_justified_epoch
if finalize:
last_finalized_epoch = crystallized_state.current_epoch - 1
dynasty = crystallized_state.dynasty + 1
new_queued_validators, new_active_validators, new_exited_validators = \
get_incremented_validator_sets(crystallized_state, new_validator_records)
else:
last_finalized_epoch = crystallized_state.last_finalized_epoch
dynasty = crystallized_state.dynasty
new_queued_validators = crystallized_state.queued_validators
new_active_validators = crystallized_state.active_validators
new_exited_validators = crystallized_state.exited_validators
crystallized_state = CrystallizedState(
queued_validators=new_queued_validators,
active_validators=new_active_validators,
exited_validators=new_exited_validators,
current_shuffling=get_shuffling(active_state.randao,
len(new_active_validators),
config=config),
last_justified_epoch=last_justified_epoch,
last_finalized_epoch=last_finalized_epoch,
dynasty=dynasty,
next_shard=0,
current_epoch=crystallized_state.current_epoch + 1,
crosslink_records=new_crosslink_records,
total_deposits=total_deposits)
# Reset the active state
active_state = ActiveState(height=active_state.height,
randao=active_state.randao,
ffg_voter_bitfield=get_empty_bitfield(
crystallized_state.num_active_validators),
balance_deltas=[],
partial_crosslinks=[],
total_skip_count=active_state.total_skip_count,
recent_proposers=[])
return crystallized_state, active_state
def test_bitfield_multiple_votes():
bitfield = get_empty_bitfield(1)
bitfield = set_voted(bitfield, 0)
bitfield = set_voted(bitfield, 0)
assert has_voted(bitfield, 0)
def test_empty_bitfield():
attesters = list(range(10))
bitfield = get_empty_bitfield(len(attesters))
for attester in attesters:
assert not has_voted(bitfield, attester)
def calculate_crosslink_rewards(
crystallized_state: CrystallizedState,
active_state: ActiveState,
block: 'Block',
config: Dict[str, Any] = DEFAULT_CONFIG) -> List[int]:
validators = crystallized_state.validators
rewards_and_penalties = [0 for _ in validators] # type: List[int]
total_deposits = crystallized_state.total_deposits
reward_quotient, quadratic_penalty_quotient = get_reward_context(
total_deposits, config)
last_state_recalc = crystallized_state.last_state_recalc
# collect crosslink participation data for each shard_id that was attempted to
# be crosslinked two cycles ago
committee_crosslinks = {} # type: Dict[str, Any]
for slot in range(max(last_state_recalc - config['cycle_length'], 0),
last_state_recalc):
shards_and_committees = get_shards_and_committees_for_slot(
crystallized_state, slot, config=config)
for shard_and_committee in shards_and_committees:
shard_id = shard_and_committee.shard_id
if shard_id not in committee_crosslinks:
committee_crosslinks[shard_id] = {
'participating_validator_indices': [],
'non_participating_validator_indices': [],
'total_participated_v_deposits': 0,
'total_v_deposits': 0
}
attestations = [
attestation
for attestation in active_state.pending_attestations if
attestation.slot == slot and attestation.shard_id == shard_id
]
if attestations:
bitfields = [
attestation.attester_bitfield
for attestation in attestations
]
bitfield = or_bitfields(bitfields)
else:
bitfield = get_empty_bitfield(
len(shard_and_committee.committee))
committee_crosslink = committee_crosslinks[shard_id]
for committee_index, validator_index in enumerate(
shard_and_committee.committee):
validator = crystallized_state.validators[validator_index]
if has_voted(bitfield, committee_index):
committee_crosslink[
'participating_validator_indices'].append(
validator_index)
committee_crosslink[
'total_participated_v_deposits'] += validator.balance
else:
committee_crosslink[
'non_participating_validator_indices'].append(
validator_index)
committee_crosslink['total_v_deposits'] += validator.balance
# for each shard and associated validator set, apply rewards/penalties based on participation
for shard_id, committee_crosslink in committee_crosslinks.items():
crosslink = crystallized_state.crosslink_records[shard_id]
if crosslink.dynasty == crystallized_state.current_dynasty:
continue
time_since_last_confirmation = block.slot_number - crosslink.slot
total_participated_v_deposits = committee_crosslink[
'total_participated_v_deposits']
total_v_deposits = committee_crosslink['total_v_deposits']
for validator_index in committee_crosslink[
'participating_validator_indices']:
validator = crystallized_state.validators[validator_index]
rewards_and_penalties[validator_index] += (
validator.balance // reward_quotient *
(2 * total_participated_v_deposits - total_v_deposits) //
total_v_deposits)
for validator_index in committee_crosslink[
'non_participating_validator_indices']:
validator = crystallized_state.validators[validator_index]
rewards_and_penalties[validator_index] -= (
(validator.balance // reward_quotient) +
(validator.balance * time_since_last_confirmation //
quadratic_penalty_quotient))
return rewards_and_penalties