def get_item(self, item_id):
if len(item_id.split('-')) != 2:
return None
session_id, rank_validator = item_id.split('-')
return SessionValidator.query(self.session).filter_by(
session_id=session_id, rank_validator=rank_validator).first()
def get_relationships(self, include_list, item):
relationships = {}
if 'blocks' in include_list:
relationships['blocks'] = Block.query(self.session).filter_by(
session_id=item.id).order_by(Block.id.desc())
if 'validators' in include_list:
relationships['validators'] = SessionValidator.query(
self.session).filter_by(session_id=item.id).order_by(
SessionValidator.rank_validator)
return relationships
def add_session(self, db_session, session_id):
current_era = None
validators = []
nominators = []
validation_session_lookup = {}
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
# Retrieve current era
storage_call = RuntimeStorage.query(db_session).filter_by(
module_id='staking',
name='CurrentEra',
spec_version=self.block.spec_version_id
).first()
if storage_call:
try:
current_era = substrate.get_storage(
block_hash=self.block.hash,
module="Staking",
function="CurrentEra",
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher
)
except RemainingScaleBytesNotEmptyException:
pass
# Retrieve validators for new session from storage
storage_call = RuntimeStorage.query(db_session).filter_by(
module_id='session',
name='Validators',
spec_version=self.block.spec_version_id
).first()
if storage_call:
try:
validators = substrate.get_storage(
block_hash=self.block.hash,
module="Session",
function="Validators",
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher
) or []
except RemainingScaleBytesNotEmptyException:
pass
# Retrieve all sessions in one call
if not LEGACY_SESSION_VALIDATOR_LOOKUP:
# Retrieve session account
# TODO move to network specific data types
storage_call = RuntimeStorage.query(db_session).filter_by(
module_id='session',
name='QueuedKeys',
spec_version=self.block.spec_version_id
).first()
if storage_call:
try:
validator_session_list = substrate.get_storage(
block_hash=self.block.hash,
module="Session",
function="QueuedKeys",
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher
) or []
except RemainingScaleBytesNotEmptyException:
try:
validator_session_list = substrate.get_storage(
block_hash=self.block.hash,
module="Session",
function="QueuedKeys",
return_scale_type='Vec<(ValidatorId, LegacyKeys)>',
hasher=storage_call.type_hasher
) or []
except RemainingScaleBytesNotEmptyException:
validator_session_list = substrate.get_storage(
block_hash=self.block.hash,
module="Session",
function="QueuedKeys",
return_scale_type='Vec<(ValidatorId, EdgewareKeys)>',
hasher=storage_call.type_hasher
) or []
# build lookup dict
validation_session_lookup = {}
for validator_session_item in validator_session_list:
session_key = ''
if validator_session_item['keys'].get('grandpa'):
session_key = validator_session_item['keys'].get('grandpa')
if validator_session_item['keys'].get('ed25519'):
session_key = validator_session_item['keys'].get('ed25519')
validation_session_lookup[
validator_session_item['validator'].replace('0x', '')] = session_key.replace('0x', '')
for rank_nr, validator_account in enumerate(validators):
validator_stash = None
validator_controller = None
validator_ledger = {}
validator_prefs = {}
validator_session = ''
exposure = {}
if not LEGACY_SESSION_VALIDATOR_LOOKUP:
validator_stash = validator_account.replace('0x', '')
# Retrieve stash account
storage_call = RuntimeStorage.query(db_session).filter_by(
module_id='staking',
name='Bonded',
spec_version=self.block.spec_version_id
).first()
if storage_call:
try:
validator_controller = substrate.get_storage(
block_hash=self.block.hash,
module="Staking",
function="Bonded",
params=validator_stash,
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher
) or ''
validator_controller = validator_controller.replace('0x', '')
except RemainingScaleBytesNotEmptyException:
pass
# Retrieve session account
validator_session = validation_session_lookup.get(validator_stash)
else:
validator_controller = validator_account.replace('0x', '')
# Retrieve stash account
storage_call = RuntimeStorage.query(db_session).filter_by(
module_id='staking',
name='Ledger',
spec_version=self.block.spec_version_id
).first()
if storage_call:
try:
validator_ledger = substrate.get_storage(
block_hash=self.block.hash,
module="Staking",
function="Ledger",
params=validator_controller,
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher
) or {}
validator_stash = validator_ledger.get('stash', '').replace('0x', '')
except RemainingScaleBytesNotEmptyException:
pass
# Retrieve session account
storage_call = RuntimeStorage.query(db_session).filter_by(
module_id='session',
name='NextKeyFor',
spec_version=self.block.spec_version_id
).first()
if storage_call:
try:
validator_session = substrate.get_storage(
block_hash=self.block.hash,
module="Session",
function="NextKeyFor",
params=validator_controller,
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher
) or ''
except RemainingScaleBytesNotEmptyException:
pass
validator_session = validator_session.replace('0x', '')
# Retrieve validator preferences for stash account
storage_call = RuntimeStorage.query(db_session).filter_by(
module_id='staking',
name='Validators',
spec_version=self.block.spec_version_id
).first()
if storage_call:
try:
validator_prefs = substrate.get_storage(
block_hash=self.block.hash,
module="Staking",
function="Validators",
params=validator_stash,
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher
) or {'col1': {}, 'col2': {}}
except RemainingScaleBytesNotEmptyException:
pass
# Retrieve nominators
storage_call = RuntimeStorage.query(db_session).filter_by(
module_id='staking',
name='Stakers',
spec_version=self.block.spec_version_id
).first()
if storage_call:
try:
exposure = substrate.get_storage(
block_hash=self.block.hash,
module="Staking",
function="Stakers",
params=validator_stash,
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher
) or {}
except RemainingScaleBytesNotEmptyException:
pass
if exposure.get('total'):
bonded_nominators = exposure.get('total') - exposure.get('own')
else:
bonded_nominators = None
session_validator = SessionValidator(
session_id=session_id,
validator_controller=validator_controller,
validator_stash=validator_stash,
bonded_total=exposure.get('total'),
bonded_active=validator_ledger.get('active'),
bonded_own=exposure.get('own'),
bonded_nominators=bonded_nominators,
validator_session=validator_session,
rank_validator=rank_nr,
unlocking=validator_ledger.get('unlocking'),
count_nominators=len(exposure.get('others', [])),
unstake_threshold=validator_prefs.get('col1', {}).get('unstakeThreshold'),
commission=validator_prefs.get('col1', {}).get('validatorPayment')
)
session_validator.save(db_session)
# Store nominators
for rank_nominator, nominator_info in enumerate(exposure.get('others', [])):
nominator_stash = nominator_info.get('who').replace('0x', '')
nominators.append(nominator_stash)
session_nominator = SessionNominator(
session_id=session_id,
rank_validator=rank_nr,
rank_nominator=rank_nominator,
nominator_stash=nominator_stash,
bonded=nominator_info.get('value'),
)
session_nominator.save(db_session)
# Store session
session = Session(
id=session_id,
start_at_block=self.block.id + 1,
created_at_block=self.block.id,
created_at_extrinsic=self.event.extrinsic_idx,
created_at_event=self.event.event_idx,
count_validators=len(validators),
count_nominators=len(set(nominators)),
era=current_era
)
session.save(db_session)
# Retrieve previous session to calculate count_blocks
prev_session = Session.query(db_session).filter_by(id=session_id - 1).first()
if prev_session:
count_blocks = self.block.id - prev_session.start_at_block + 1
else:
count_blocks = self.block.id
session_total = SessionTotal(
id=session_id - 1,
end_at_block=self.block.id,
count_blocks=count_blocks
)
session_total.save(db_session)
def get_query(self):
return SessionValidator.query(self.session).order_by(
SessionValidator.session_id, SessionValidator.rank_validator)
def add_session(self, db_session, session_id):
nominators = []
# Retrieve current era
try:
current_era = self.substrate.get_runtime_state(
module="Staking",
storage_function="CurrentEra",
params=[],
block_hash=self.block.hash
).get('result')
except StorageFunctionNotFound:
current_era = None
# Retrieve validators for new session from storage
try:
validators = self.substrate.get_runtime_state(
module="Session",
storage_function="Validators",
params=[],
block_hash=self.block.hash
).get('result', [])
except StorageFunctionNotFound:
validators = []
for rank_nr, validator_account in enumerate(validators):
validator_ledger = {}
validator_session = None
validator_stash = validator_account.replace('0x', '')
# Retrieve controller account
try:
validator_controller = self.substrate.get_runtime_state(
module="Staking",
storage_function="Bonded",
params=[validator_account],
block_hash=self.block.hash
).get('result')
if validator_controller:
validator_controller = validator_controller.replace('0x', '')
except StorageFunctionNotFound:
validator_controller = None
# Retrieve validator preferences for stash account
try:
validator_prefs = self.substrate.get_runtime_state(
module="Staking",
storage_function="ErasValidatorPrefs",
params=[current_era, validator_account],
block_hash=self.block.hash
).get('result')
except StorageFunctionNotFound:
validator_prefs = None
if not validator_prefs:
validator_prefs = {'commission': None}
# Retrieve bonded
try:
exposure = self.substrate.get_runtime_state(
module="Staking",
storage_function="ErasStakers",
params=[current_era, validator_account],
block_hash=self.block.hash
).get('result')
except StorageFunctionNotFound:
exposure = None
if not exposure:
exposure = {}
if exposure.get('total'):
bonded_nominators = exposure.get('total') - exposure.get('own')
else:
bonded_nominators = None
session_validator = SessionValidator(
session_id=session_id,
validator_controller=validator_controller,
validator_stash=validator_stash,
bonded_total=exposure.get('total'),
bonded_active=validator_ledger.get('active'),
bonded_own=exposure.get('own'),
bonded_nominators=bonded_nominators,
validator_session=validator_session,
rank_validator=rank_nr,
unlocking=validator_ledger.get('unlocking'),
count_nominators=len(exposure.get('others', [])),
unstake_threshold=None,
commission=validator_prefs.get('commission')
)
session_validator.save(db_session)
# Store nominators
for rank_nominator, nominator_info in enumerate(exposure.get('others', [])):
nominator_stash = nominator_info.get('who').replace('0x', '')
nominators.append(nominator_stash)
session_nominator = SessionNominator(
session_id=session_id,
rank_validator=rank_nr,
rank_nominator=rank_nominator,
nominator_stash=nominator_stash,
bonded=nominator_info.get('value'),
)
session_nominator.save(db_session)
# Store session
session = Session(
id=session_id,
start_at_block=self.block.id + 1,
created_at_block=self.block.id,
created_at_extrinsic=self.event.extrinsic_idx,
created_at_event=self.event.event_idx,
count_validators=len(validators),
count_nominators=len(set(nominators)),
era=current_era
)
session.save(db_session)
# Retrieve previous session to calculate count_blocks
prev_session = Session.query(db_session).filter_by(id=session_id - 1).first()
if prev_session:
count_blocks = self.block.id - prev_session.start_at_block + 1
else:
count_blocks = self.block.id
session_total = SessionTotal(
id=session_id - 1,
end_at_block=self.block.id,
count_blocks=count_blocks
)
session_total.save(db_session)
# Update validator flags
validator_ids = [v.replace('0x', '') for v in validators]
Account.query(db_session).filter(
Account.id.in_(validator_ids), Account.was_validator == False
).update({Account.was_validator: True}, synchronize_session='fetch')
Account.query(db_session).filter(
Account.id.notin_(validator_ids), Account.is_validator == True
).update({Account.is_validator: False}, synchronize_session='fetch')
Account.query(db_session).filter(
Account.id.in_(validator_ids), Account.is_validator == False
).update({Account.is_validator: True}, synchronize_session='fetch')
# Update nominator flags
Account.query(db_session).filter(
Account.id.in_(nominators), Account.was_nominator == False
).update({Account.was_nominator: True}, synchronize_session='fetch')
Account.query(db_session).filter(
Account.id.notin_(nominators), Account.is_nominator == True
).update({Account.is_nominator: False}, synchronize_session='fetch')
Account.query(db_session).filter(
Account.id.in_(nominators), Account.is_nominator == False
).update({Account.is_nominator: True}, synchronize_session='fetch')
def sequencing_hook(self, db_session, parent_block_data,
parent_sequenced_block_data):
if not parent_sequenced_block_data:
parent_sequenced_block_data = {}
if parent_block_data and parent_block_data['datetime']:
self.sequenced_block.parent_datetime = parent_block_data[
'datetime']
if type(parent_block_data['datetime']) is str:
self.sequenced_block.blocktime = (
self.block.datetime - dateutil.parser.parse(
parent_block_data['datetime'])).total_seconds()
else:
self.sequenced_block.blocktime = (
self.block.datetime -
parent_block_data['datetime']).total_seconds()
else:
self.sequenced_block.blocktime = 0
self.sequenced_block.parent_datetime = self.block.datetime
self.sequenced_block.total_extrinsics = int(
parent_sequenced_block_data.get('total_extrinsics',
0)) + self.block.count_extrinsics
self.sequenced_block.total_extrinsics_success = int(
parent_sequenced_block_data.get(
'total_extrinsics_success',
0)) + self.block.count_extrinsics_success
self.sequenced_block.total_extrinsics_error = int(
parent_sequenced_block_data.get(
'total_extrinsics_error',
0)) + self.block.count_extrinsics_error
self.sequenced_block.total_extrinsics_signed = int(
parent_sequenced_block_data.get(
'total_extrinsics_signed',
0)) + self.block.count_extrinsics_signed
self.sequenced_block.total_extrinsics_unsigned = int(
parent_sequenced_block_data.get(
'total_extrinsics_unsigned',
0)) + self.block.count_extrinsics_unsigned
self.sequenced_block.total_extrinsics_signedby_address = int(
parent_sequenced_block_data.get(
'total_extrinsics_signedby_address',
0)) + self.block.count_extrinsics_signedby_address
self.sequenced_block.total_extrinsics_signedby_index = int(
parent_sequenced_block_data.get(
'total_extrinsics_signedby_index',
0)) + self.block.count_extrinsics_signedby_index
self.sequenced_block.total_events = int(
parent_sequenced_block_data.get('total_events',
0)) + self.block.count_events
self.sequenced_block.total_events_system = int(
parent_sequenced_block_data.get(
'total_events_system', 0)) + self.block.count_events_system
self.sequenced_block.total_events_module = int(
parent_sequenced_block_data.get(
'total_events_module', 0)) + self.block.count_events_module
self.sequenced_block.total_events_extrinsic = int(
parent_sequenced_block_data.get(
'total_events_extrinsic',
0)) + self.block.count_events_extrinsic
self.sequenced_block.total_events_finalization = int(
parent_sequenced_block_data.get(
'total_events_finalization',
0)) + self.block.count_events_finalization
self.sequenced_block.total_blocktime = int(
parent_sequenced_block_data.get(
'total_blocktime', 0)) + self.sequenced_block.blocktime
self.sequenced_block.total_accounts_new = int(
parent_sequenced_block_data.get('total_accounts_new',
0)) + self.block.count_accounts_new
self.sequenced_block.total_logs = int(
parent_sequenced_block_data.get('total_logs',
0)) + self.block.count_log
self.sequenced_block.total_accounts = int(
parent_sequenced_block_data.get('total_accounts',
0)) + self.block.count_accounts
self.sequenced_block.total_accounts_reaped = int(
parent_sequenced_block_data.get(
'total_accounts_reaped', 0)) + self.block.count_accounts_reaped
self.sequenced_block.total_sessions_new = int(
parent_sequenced_block_data.get('total_sessions_new',
0)) + self.block.count_sessions_new
self.sequenced_block.total_contracts_new = int(
parent_sequenced_block_data.get(
'total_contracts_new', 0)) + self.block.count_contracts_new
self.sequenced_block.session_id = int(
parent_sequenced_block_data.get('session_id', 0))
if parent_block_data and parent_block_data['count_sessions_new'] > 0:
self.sequenced_block.session_id += 1
if self.block.slot_number is not None:
if self.block.authority_index is not None:
rank_validator = self.block.authority_index
else:
# In case of AURA, validator slot is determined by unique slot number
validator_count = SessionValidator.query(db_session).filter_by(
session_id=self.sequenced_block.session_id).count()
rank_validator = int(self.block.slot_number) % validator_count
# Retrieve block producer from session validator set
validator = SessionValidator.query(db_session).filter_by(
session_id=self.sequenced_block.session_id,
rank_validator=rank_validator).first()
if validator:
self.sequenced_block.author = validator.validator_stash