def on_get(self, req, resp, network_id=None):
resp.status = falcon.HTTP_200
# TODO make caching more generic for custom resources
cache_key = '{}-{}'.format(req.method, req.url)
console_handler = logging.StreamHandler()
console_handler.setLevel('INFO')
logger = logging.getLogger('yee')
logger.setLevel('INFO')
logger.addHandler(console_handler)
# logger.info(cache_key)
response = self.cache_region.get(cache_key, self.cache_expiration_time)
if response is NO_VALUE:
substrate01 = SubstrateInterface(SHARDS_TABLE['shard.0'])
substrate02 = SubstrateInterface(SHARDS_TABLE['shard.1'])
substrate03 = SubstrateInterface(SHARDS_TABLE['shard.2'])
substrate04 = SubstrateInterface(SHARDS_TABLE['shard.3'])
shard01 = substrate01.get_block_header(None)
shard01['finalizedNum'] = substrate01.get_block_number(
substrate01.get_chain_finalised_head())
shard02 = substrate02.get_block_header(None)
shard02['finalizedNum'] = substrate02.get_block_number(
substrate02.get_chain_finalised_head())
shard03 = substrate03.get_block_header(None)
shard03['finalizedNum'] = substrate03.get_block_number(
substrate03.get_chain_finalised_head())
shard04 = substrate04.get_block_header(None)
shard04['finalizedNum'] = substrate04.get_block_number(
substrate04.get_chain_finalised_head())
response = self.get_jsonapi_response(data={
'type': 'FinalizedHeadList',
'attributes': {
'shard01': shard01,
'shard02': shard02,
'shard03': shard03,
'shard04': shard04
}
}, )
self.cache_region.set(cache_key, response)
resp.set_header('X-Cache', 'MISS')
else:
resp.set_header('X-Cache', 'HIT')
resp.media = response
def on_get(self, req, resp):
sequencer_task = Status.get_status(self.session, 'SEQUENCER_TASK_ID')
integrity_head = Status.get_status(self.session, 'INTEGRITY_HEAD')
sequencer_head = self.session.query(func.max(BlockTotal.id)).one()[0]
best_block = Block.query(self.session).filter_by(
id=self.session.query(func.max(Block.id)).one()[0]).first()
if best_block:
best_block_datetime = best_block.datetime.replace(tzinfo=pytz.UTC).timestamp() * 1000
best_block_nr = best_block.id
else:
best_block_datetime = None
best_block_nr = None
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
chain_head_block_id = substrate.get_block_number(substrate.get_chain_head())
chain_finalized_block_id = substrate.get_block_number(substrate.get_chain_finalised_head())
resp.media = {
'best_block_datetime': best_block_datetime,
'best_block_nr': best_block_nr,
'sequencer_task': sequencer_task.value,
'sequencer_head': sequencer_head,
'integrity_head': int(integrity_head.value),
'chain_head_block_id': chain_head_block_id,
'chain_finalized_block_id': chain_finalized_block_id
}
def balance_snapshot(self, account_id=None, block_start=1, block_end=None, block_ids=None):
if account_id:
accounts = [account_id]
else:
accounts = [account.id for account in Account.query(self.session)]
harvester = PolkascanHarvesterService(
db_session=self.session,
type_registry=TYPE_REGISTRY,
type_registry_file=TYPE_REGISTRY_FILE
)
if block_ids:
block_range = block_ids
else:
if block_end is None:
# Set block end to chaintip
substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL, runtime_config=RuntimeConfiguration())
block_end = substrate.get_block_number(substrate.get_chain_finalised_head())
block_range = range(block_start, block_end + 1)
for block_id in block_range:
for account in accounts:
harvester.create_balance_snapshot(block_id, account)
self.session.commit()
return {
'message': 'Snapshop created',
'account_id': account_id,
'block_start': block_start,
'block_end': block_end,
'block_ids': block_ids
}
def start_harvester(self, check_gaps=False):
substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL,
type_registry_preset=settings.TYPE_REGISTRY,
runtime_config=RuntimeConfiguration())
block_sets = []
if check_gaps:
# Check for gaps between already harvested blocks and try to fill them first
remaining_sets_result = Block.get_missing_block_ids(self.session)
for block_set in remaining_sets_result:
# Get start and end block hash
end_block_hash = substrate.get_block_hash(
int(block_set['block_from']))
start_block_hash = substrate.get_block_hash(
int(block_set['block_to']))
# Start processing task
accumulate_block_recursive.delay(start_block_hash, end_block_hash)
block_sets.append({
'start_block_hash': start_block_hash,
'end_block_hash': end_block_hash
})
# Start sequencer
sequencer_task = start_sequencer.delay()
# Continue from current (finalised) head
if FINALIZATION_ONLY == 1:
start_block_hash = substrate.get_chain_finalised_head()
else:
start_block_hash = substrate.get_chain_head()
end_block_hash = None
accumulate_block_recursive.delay(start_block_hash, end_block_hash)
block_sets.append({
'start_block_hash': start_block_hash,
'end_block_hash': end_block_hash
})
return {
'result': 'Harvester job started',
'block_sets': block_sets,
'sequencer_task_id': sequencer_task.task_id
}
def on_get(self, req, resp):
substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL,
address_type=SUBSTRATE_ADDRESS_TYPE,
type_registry_preset=TYPE_REGISTRY)
resp.status = falcon.HTTP_200
# head = Block.get_head(self.session);
head_hash = substrate.get_chain_head()
head_number = substrate.get_block_number(head_hash)
finalised_head_hash = substrate.get_chain_finalised_head()
finalised_head_number = substrate.get_block_number(finalised_head_hash)
extrinsicCount = Extrinsic.query(self.session).count()
try:
validators = substrate.get_runtime_state(
module="Session",
storage_function="Validators",
params=[],
block_hash=head_hash).get('result', [])
except StorageFunctionNotFound:
validators = []
try:
validator_count = substrate.get_runtime_state(
module="Staking",
storage_function="ValidatorCount",
params=[],
block_hash=head_hash).get('result', 0)
except StorageFunctionNotFound:
validator_count = 0
transfers_count = Extrinsic.query(self.session).filter(
and_(Extrinsic.module_id == 'balances',
Extrinsic.call_id == 'transfer')).count()
resp.media = {
'status': 'success',
'data': {
'blockNumber': head_number,
'finalizedBlockNumber': finalised_head_number,
'extrinsics': extrinsicCount,
'currValidators': len(validators),
'validators': validator_count,
'transfersCount': transfers_count
}
}
def get_balance_for_address(address):
"""
Params:
-------
address - str
Returns:
--------
dict
Example:
{
"free": 22000000000,
"reserved": 0,
"miscFrozen": 0,
"feeFrozen": 0,
}
"""
substrate = SubstrateInterface(
url=settings.NODE_URL,
address_type=2,
type_registry_preset="kusama",
)
block = substrate.get_chain_finalised_head()
account_data = substrate.get_runtime_state(
module="System",
storage_function="Account",
params=[address],
block_hash=block,
)
if not account_data.get("result"):
return {
"free": 0,
"reserved": 0,
"miscFrozen": 0,
"feeFrozen": 0,
}
return account_data.get("result", {}).get("data")
def main(block_hash=None):
substrate = SubstrateInterface(
url="ws://127.0.0.1:9944/",
address_type=32,
type_registry=TYPES,
)
result = substrate.get_runtime_block(block_hash=block_hash)
chain_finalised_head = substrate.get_chain_finalised_head()
result = substrate.get_runtime_block(block_hash=chain_finalised_head)
print("%i extrinsics in block %i" % (len(
result['block']['extrinsics']), result['block']['header']['number']))
# for extrinsic in result['block']['extrinsics']:
# # print(extrinsic)
# if 'account_id' in extrinsic:
# signed_by_address = ss58_encode(address=extrinsic['account_id'], address_type=32)
# else:
# signed_by_address = None
# print('\nModule: {}\nCall: {}\nSigned by: {}'.format(
# extrinsic['call_module'],
# extrinsic['call_function'],
# signed_by_address
# ))
# for param in extrinsic['params']:
# if param['type'] == 'Address':
# param['value'] = ss58_encode(address=param['value'], address_type=32)
# if param['type'] == 'Compact<Balance>':
# param['value'] = '{} TXRT'.format(param['value'] / 10**12)
# print("Param '{}': {}".format(param['name'], param['value']))
# print(len(result['block']['extrinsics']))
return None
def integrity_checks(self):
# 1. Check finalized head
substrate = SubstrateInterface(settings.SUBSTRATE_RPC_URL)
if settings.FINALIZATION_BY_BLOCK_CONFIRMATIONS > 0:
finalized_block_hash = substrate.get_chain_head()
finalized_block_number = max(
substrate.get_block_number(finalized_block_hash) - settings.FINALIZATION_BY_BLOCK_CONFIRMATIONS, 0
)
else:
finalized_block_hash = substrate.get_chain_finalised_head()
finalized_block_number = substrate.get_block_number(finalized_block_hash)
# 2. Check integrity head
integrity_head = Status.get_status(self.db_session, 'INTEGRITY_HEAD')
if not integrity_head.value:
# Only continue if block #1 exists
if Block.query(self.db_session).filter_by(id=1).count() == 0:
raise BlockIntegrityError('Chain not at genesis')
integrity_head.value = 0
else:
integrity_head.value = int(integrity_head.value)
start_block_id = max(integrity_head.value - 1, 0)
end_block_id = finalized_block_number
chunk_size = 1000
parent_block = None
if start_block_id < end_block_id:
# Continue integrity check
# print('== Start integrity checks from {} to {} =='.format(start_block_id, end_block_id))
for block_nr in range(start_block_id, end_block_id, chunk_size):
# TODO replace limit with filter_by block range
block_range = Block.query(self.db_session).order_by('id')[block_nr:block_nr + chunk_size]
for block in block_range:
if parent_block:
if block.id != parent_block.id + 1:
# Save integrity head if block hash of parent matches with hash in node
if parent_block.hash == substrate.get_block_hash(integrity_head.value):
integrity_head.save(self.db_session)
self.db_session.commit()
raise BlockIntegrityError('Block #{} is missing.. stopping check '.format(parent_block.id + 1))
elif block.parent_hash != parent_block.hash:
self.process_reorg_block(parent_block)
self.process_reorg_block(block)
self.remove_block(block.hash)
self.remove_block(parent_block.hash)
self.db_session.commit()
self.add_block(substrate.get_block_hash(block.id))
self.add_block(substrate.get_block_hash(parent_block.id))
self.db_session.commit()
integrity_head.value = parent_block.id - 1
# Save integrity head if block hash of parent matches with hash in node
#if parent_block.parent_hash == substrate.get_block_hash(integrity_head.value):
integrity_head.save(self.db_session)
self.db_session.commit()
raise BlockIntegrityError('Block #{} failed integrity checks, Re-adding #{}.. '.format(parent_block.id, block.id))
else:
integrity_head.value = block.id
parent_block = block
if block.id == end_block_id:
break
if parent_block:
if parent_block.hash == substrate.get_block_hash(int(integrity_head.value)):
integrity_head.save(self.db_session)
self.db_session.commit()
return {'integrity_head': integrity_head.value}
'types': {
"StakingLedger<AccountId, BalanceOf>": {
"type":
"struct",
"type_mapping": [["stash", "AccountId"],
["total", "Compact<Balance>"],
["active", "Compact<Balance>"],
["unlocking", "Vec<UnlockChunk<Balance>>"],
["claimedReward", "Vec<EraIndex>"]]
},
}
},
type_registry_preset='polkadot',
)
head = substrate.get_chain_finalised_head()
def get_candidates():
prefix = substrate.generate_storage_hash("Staking", "Validators")
pairs = substrate.rpc_request(method="state_getPairs",
params=[prefix, head])['result']
last_32_bytes = list(map(lambda p: "0x" + p[0][-64:], pairs))
return list(map(lambda k: ss58_encode(k), last_32_bytes))
def get_nominators():
prefix = substrate.generate_storage_hash("Staking", "Nominators")
pairs = substrate.rpc_request(method="state_getPairs",
params=[prefix, head])['result']
