def start_harvester(self, check_gaps=False):
substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL,
type_registry_preset=settings.TYPE_REGISTRY,
runtime_config=RuntimeConfiguration())
block_sets = []
if check_gaps:
# Check for gaps between already harvested blocks and try to fill them first
remaining_sets_result = Block.get_missing_block_ids(self.session)
for block_set in remaining_sets_result:
# Get start and end block hash
end_block_hash = substrate.get_block_hash(
int(block_set['block_from']))
start_block_hash = substrate.get_block_hash(
int(block_set['block_to']))
# Start processing task
accumulate_block_recursive.delay(start_block_hash, end_block_hash)
block_sets.append({
'start_block_hash': start_block_hash,
'end_block_hash': end_block_hash
})
# Start sequencer
sequencer_task = start_sequencer.delay()
# Continue from current (finalised) head
if FINALIZATION_ONLY == 1:
start_block_hash = substrate.get_chain_finalised_head()
else:
start_block_hash = substrate.get_chain_head()
end_block_hash = None
accumulate_block_recursive.delay(start_block_hash, end_block_hash)
block_sets.append({
'start_block_hash': start_block_hash,
'end_block_hash': end_block_hash
})
return {
'result': 'Harvester job started',
'block_sets': block_sets,
'sequencer_task_id': sequencer_task.task_id
}
def start_init(self):
if self.init:
print('start_init task is running : '.format(self.init))
return {'result': 'waiting init task completed! '}
n = Block.query(self.session).filter_by(bid=1).count()
if n >= 4:
print(' init task is completed! count().n: {} '.format(n))
return {'result': ' init task is completed! '}
self.init = True
print("start_init")
for shard in SHARDS_TABLE:
substrate_url = SHARDS_TABLE[shard]
substrate = SubstrateInterface(substrate_url)
start_block_hash = substrate.get_block_hash(3)
print('== start_init substrate_url {} ==start_block_hash-{}'.format(
substrate_url, start_block_hash))
end_block_hash = None
accumulate_block_recursive.delay(start_block_hash, end_block_hash,
substrate_url)
return {
'result': 'start_init job started',
'SHARDS_TABLE': SHARDS_TABLE,
'init_task_id': self.request.id
}
def on_post(self, req, resp):
block_hash = None
if req.media.get('block_id'):
substrate = SubstrateInterface(
url=SUBSTRATE_RPC_URL,
runtime_config=RuntimeConfiguration(),
type_registry_preset=settings.TYPE_REGISTRY)
block_hash = substrate.get_block_hash(req.media.get('block_id'))
elif req.media.get('block_hash'):
block_hash = req.media.get('block_hash')
else:
resp.status = falcon.HTTP_BAD_REQUEST
resp.media = {
'errors': ['Either block_hash or block_id should be supplied']
}
if block_hash:
print('Processing {} ...'.format(block_hash))
harvester = PolkascanHarvesterService(
db_session=self.session,
type_registry=TYPE_REGISTRY,
type_registry_file=TYPE_REGISTRY_FILE + '-mst')
block = Block.query(
self.session).filter(Block.hash == block_hash).first()
if block:
resp.status = falcon.HTTP_200
resp.media = {
'result': 'already exists',
'parentHash': block.parent_hash
}
else:
amount = req.media.get('amount', 1)
for nr in range(0, amount):
try:
block = harvester.add_block(block_hash)
except BlockAlreadyAdded as e:
print('Skipping {}'.format(block_hash))
block_hash = block.parent_hash
if block.id == 0:
break
self.session.commit()
resp.status = falcon.HTTP_201
resp.media = {
'result': 'added',
'parentHash': block.parent_hash
}
else:
resp.status = falcon.HTTP_404
resp.media = {'result': 'Block not found'}
def start_harvester(self, check_gaps=False):
print("---------- {}".format(check_gaps))
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
block_sets = []
if check_gaps:
# Check for gaps between already harvested blocks and try to fill them first
remaining_sets_result = Block.get_missing_block_ids(self.session)
for block_set in remaining_sets_result:
# Get start and end block hash
end_block_hash = substrate.get_block_hash(
int(block_set['block_from']))
start_block_hash = substrate.get_block_hash(
int(block_set['block_to']))
# Start processing task
accumulate_block_recursive.delay(start_block_hash, end_block_hash)
block_sets.append({
'start_block_hash': start_block_hash,
'end_block_hash': end_block_hash
})
# Start sequencer
start_sequencer.delay()
# Continue from current finalised head
start_block_hash = substrate.get_chain_head()
end_block_hash = None
accumulate_block_recursive.delay(start_block_hash, end_block_hash)
block_sets.append({
'start_block_hash': start_block_hash,
'end_block_hash': end_block_hash
})
return {'result': 'Harvester job started', 'block_sets': block_sets}
def result_handler(result):
if result.get("method", "") == "chain_newHead":
number = result['params']['result']['number']
substrate_connection = SubstrateInterface(url=rpc_url)
block_hash = substrate_connection.get_block_hash(number)
block_events = substrate_connection.get_runtime_events(
block_hash=block_hash).get('result')
print(int(number, 16), block_hash)
for block_event in block_events:
module_id = block_event['module_id']
event_id = block_event['event_id']
if module_id == "Staking" and event_id == "EraPayout":
print(f"{module_id}.{event_id}")
publisher.publish(topic_path, b' ')
def get_extrinsic_events(extrinsic_time_point):
substrate = SubstrateInterface(
url=settings.NODE_URL,
address_type=2,
type_registry_preset="kusama",
)
extrinsic_events = []
block_hash = substrate.get_block_hash(extrinsic_time_point[0])
block_events = substrate.get_runtime_events(block_hash).get("result")
for event in block_events:
if event["extrinsic_idx"] == extrinsic_time_point[1]:
# Todo, only get required event data
# Perhaps just event_id and params?
extrinsic_events.append(event)
return extrinsic_events
def on_post(self, req, resp):
blockHash = None
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
if req.media.get('block_id'):
blockHash = substrate.get_block_hash(req.media.get('block_id'))
elif req.media.get('block_hash'):
blockHash = req.media.get('block_hash')
else:
resp.status = falcon.HTTP_BAD_REQUEST
resp.media = {
'errors': ['Either blockHash or block_id should be supplied']
}
metadata = substrate.get_block_metadata(blockHash)
resp.media = {'status': 'success', 'data': metadata}
def solve(self, block_id, end_block_id, substrate_url):
print('== updatelog ===substrate_url-{} ===start_block_hash-{} ===end-{}'.format(substrate_url, block_id,
end_block_id))
substrate = SubstrateInterface(substrate_url)
try:
for nr in range(int(block_id), int(end_block_id)):
block_hash = substrate.get_block_hash(hex(nr))
json_block = substrate.get_chain_block(block_hash)
l_block_id = json_block['block']['header'].pop('number')
l_block_id = int(l_block_id, 16)
digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None)
print('== digest_logs ===digest_logs-{} '.format(digest_logs))
print('== l_block_id ===l_block_id-{} '.format(l_block_id))
self.log_processor(digest_logs, l_block_id)
except BlockAlreadyAdded as e:
print('. Skipped {} '.format(block_hash))
def start_harvester(self, check_gaps=False, shard=None):
shard = self.request.args[0]
if shard is None:
raise HarvesterNotshardParamsError(
'params shard is missing.. stopping harvester ')
print("start_harvester")
substrate_url = SHARDS_TABLE[shard]
print('== start_harvester substrate_url {} =='.format(substrate_url))
substrate = SubstrateInterface(substrate_url)
n = Block.query(self.session).filter_by(bid=1).count()
if n < 4:
print('waiting init task completed! count().n: {} '.format(n))
return {'result': 'waiting init task completed! '}
block_sets = []
harvester = PolkascanHarvesterService(self.session,
type_registry=TYPE_REGISTRY)
harvester.metadata_store = self.metadata_store
start_block_hash = substrate.get_chain_head()
end_block_hash = None
r = 10
block_nr = substrate.get_block_number(start_block_hash)
max_block = Block.query(self.session).filter_by(
shard_num=shard.split(".")[1]).order_by(Block.bid.desc()).first()
print('start block_nr {} =='.format(block_nr))
print('start max_block {} =='.format(max_block.bid))
if block_nr - max_block.bid < 10:
r = block_nr - max_block.bid
print('current range r: {} =='.format(max_block.bid))
try:
for nr in range(1, r + 1):
block_hash = substrate.get_block_hash(max_block.bid + nr)
if harvester.add_block(block_hash, substrate_url):
print('start_harvester+ Added {} '.format(block_hash))
self.session.commit()
# Update persistent metadata store in Celery task
self.metadata_store = harvester.metadata_store
except BlockAlreadyAdded as e:
print('. Skipped {} '.format(block_hash))
except IntegrityError as e:
print('. Skipped duplicate {}=={} '.format(block_hash, e))
except Exception as exc:
print('! ERROR adding {}'.format(block_hash))
raise HarvesterCouldNotAddBlock(block_hash) from exc
block_sets.append({
'start_block_hash': start_block_hash,
'end_block_hash': end_block_hash
})
return {
'result':
'Yee data Synchronization job SUCCESS',
'block_sets':
block_sets,
'result':
'Synch data from {} to {} blocks check by shardnum of {}'.format(
max_block.bid + 1, r + max_block.bid + 1, shard)
}
def integrity_checks(self):
# 1. Check finalized head
substrate = SubstrateInterface(settings.SUBSTRATE_RPC_URL)
if settings.FINALIZATION_BY_BLOCK_CONFIRMATIONS > 0:
finalized_block_hash = substrate.get_chain_head()
finalized_block_number = max(
substrate.get_block_number(finalized_block_hash) - settings.FINALIZATION_BY_BLOCK_CONFIRMATIONS, 0
)
else:
finalized_block_hash = substrate.get_chain_finalised_head()
finalized_block_number = substrate.get_block_number(finalized_block_hash)
# 2. Check integrity head
integrity_head = Status.get_status(self.db_session, 'INTEGRITY_HEAD')
if not integrity_head.value:
# Only continue if block #1 exists
if Block.query(self.db_session).filter_by(id=1).count() == 0:
raise BlockIntegrityError('Chain not at genesis')
integrity_head.value = 0
else:
integrity_head.value = int(integrity_head.value)
start_block_id = max(integrity_head.value - 1, 0)
end_block_id = finalized_block_number
chunk_size = 1000
parent_block = None
if start_block_id < end_block_id:
# Continue integrity check
# print('== Start integrity checks from {} to {} =='.format(start_block_id, end_block_id))
for block_nr in range(start_block_id, end_block_id, chunk_size):
# TODO replace limit with filter_by block range
block_range = Block.query(self.db_session).order_by('id')[block_nr:block_nr + chunk_size]
for block in block_range:
if parent_block:
if block.id != parent_block.id + 1:
# Save integrity head if block hash of parent matches with hash in node
if parent_block.hash == substrate.get_block_hash(integrity_head.value):
integrity_head.save(self.db_session)
self.db_session.commit()
raise BlockIntegrityError('Block #{} is missing.. stopping check '.format(parent_block.id + 1))
elif block.parent_hash != parent_block.hash:
self.process_reorg_block(parent_block)
self.process_reorg_block(block)
self.remove_block(block.hash)
self.remove_block(parent_block.hash)
self.db_session.commit()
self.add_block(substrate.get_block_hash(block.id))
self.add_block(substrate.get_block_hash(parent_block.id))
self.db_session.commit()
integrity_head.value = parent_block.id - 1
# Save integrity head if block hash of parent matches with hash in node
#if parent_block.parent_hash == substrate.get_block_hash(integrity_head.value):
integrity_head.save(self.db_session)
self.db_session.commit()
raise BlockIntegrityError('Block #{} failed integrity checks, Re-adding #{}.. '.format(parent_block.id, block.id))
else:
integrity_head.value = block.id
parent_block = block
if block.id == end_block_id:
break
if parent_block:
if parent_block.hash == substrate.get_block_hash(int(integrity_head.value)):
integrity_head.save(self.db_session)
self.db_session.commit()
return {'integrity_head': integrity_head.value}
class PolkascanHarvesterService(BaseService):
def __init__(self, db_session, type_registry='default', type_registry_file=None):
self.db_session = db_session
if type_registry_file:
custom_type_registry = load_type_registry_file(type_registry_file)
else:
custom_type_registry = None
self.substrate = SubstrateInterface(
url=settings.SUBSTRATE_RPC_URL,
type_registry=custom_type_registry,
type_registry_preset=type_registry
)
self.metadata_store = {}
def process_genesis(self, block):
# Set block time of parent block
child_block = Block.query(self.db_session).filter_by(parent_hash=block.hash).first()
block.set_datetime(child_block.datetime)
# Retrieve genesis accounts
if settings.get_versioned_setting('SUBSTRATE_STORAGE_INDICES', block.spec_version_id) == 'Accounts':
# Get accounts from storage keys
storage_key_prefix = self.substrate.generate_storage_hash(
storage_module='System',
storage_function='Account',
metadata_version=settings.SUBSTRATE_METADATA_VERSION
)
rpc_result = self.substrate.rpc_request(
'state_getKeys',
[storage_key_prefix, block.hash]
).get('result')
# Extract accounts from storage key
genesis_accounts = [storage_key[-64:] for storage_key in rpc_result if len(storage_key) == 162]
for account_id in genesis_accounts:
account_audit = AccountAudit(
account_id=account_id,
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
elif settings.get_versioned_setting('SUBSTRATE_STORAGE_INDICES', block.spec_version_id) == 'EnumSet':
genesis_account_page_count = self.substrate.get_runtime_state(
module="Indices",
storage_function="NextEnumSet",
block_hash=block.hash
).get('result', 0)
# Get Accounts on EnumSet
block.count_accounts_new = 0
block.count_accounts = 0
for enum_set_nr in range(0, genesis_account_page_count + 1):
genesis_accounts = self.substrate.get_runtime_state(
module="Indices",
storage_function="EnumSet",
params=[enum_set_nr],
block_hash=block.hash
).get('result')
if genesis_accounts:
block.count_accounts_new += len(genesis_accounts)
block.count_accounts += len(genesis_accounts)
for idx, account_id in enumerate(genesis_accounts):
account_audit = AccountAudit(
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
account_index_id = enum_set_nr * 64 + idx
account_index_audit = AccountIndexAudit(
account_index_id=account_index_id,
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_INDEX_AUDIT_TYPE_NEW
)
account_index_audit.save(self.db_session)
block.save(self.db_session)
# Add hardcoded account like treasury stored in settings
for account_id in settings.SUBSTRATE_TREASURY_ACCOUNTS:
account_audit = AccountAudit(
account_id=account_id,
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
data={'is_treasury': True},
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
# Check for sudo accounts
try:
# Update sudo key
sudo_key = self.substrate.get_runtime_state(
module='Sudo',
storage_function='Key',
block_hash=block.hash
).get('result')
account_audit = AccountAudit(
account_id=sudo_key.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
data={'is_sudo': True},
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
except ValueError:
pass
# Create initial session
initial_session_event = NewSessionEventProcessor(
block=block, event=Event(), substrate=self.substrate
)
if settings.get_versioned_setting('NEW_SESSION_EVENT_HANDLER', block.spec_version_id):
initial_session_event.add_session(db_session=self.db_session, session_id=0)
else:
initial_session_event.add_session_old(db_session=self.db_session, session_id=0)
def process_metadata(self, spec_version, block_hash):
# Check if metadata already in store
if spec_version not in self.metadata_store:
print('Metadata: CACHE MISS', spec_version)
runtime_version_data = self.substrate.get_block_runtime_version(block_hash)
runtime = Runtime.query(self.db_session).get(spec_version)
if runtime:
self.metadata_store[spec_version] = self.substrate.get_block_metadata(block_hash=block_hash)
else:
self.db_session.begin(subtransactions=True)
try:
# Store metadata in database
runtime = Runtime(
id=spec_version,
impl_name=runtime_version_data["implName"],
impl_version=runtime_version_data["implVersion"],
spec_name=runtime_version_data["specName"],
spec_version=spec_version,
json_metadata=str(self.substrate.metadata_decoder.data),
json_metadata_decoded=self.substrate.metadata_decoder.value,
apis=runtime_version_data["apis"],
authoring_version=runtime_version_data["authoringVersion"],
count_call_functions=0,
count_events=0,
count_modules=len(self.substrate.metadata_decoder.metadata.modules),
count_storage_functions=0,
count_constants=0,
count_errors=0
)
runtime.save(self.db_session)
print('store version to db', self.substrate.metadata_decoder.version)
for module in self.substrate.metadata_decoder.metadata.modules:
# Check if module exists
if RuntimeModule.query(self.db_session).filter_by(
spec_version=spec_version,
module_id=module.get_identifier()
).count() == 0:
module_id = module.get_identifier()
else:
module_id = '{}_1'.format(module.get_identifier())
# Storage backwards compt check
if module.storage and isinstance(module.storage, list):
storage_functions = module.storage
elif module.storage and isinstance(getattr(module.storage, 'value'), dict):
storage_functions = module.storage.items
else:
storage_functions = []
runtime_module = RuntimeModule(
spec_version=spec_version,
module_id=module_id,
prefix=module.prefix,
name=module.name,
count_call_functions=len(module.calls or []),
count_storage_functions=len(storage_functions),
count_events=len(module.events or []),
count_constants=len(module.constants or []),
count_errors=len(module.errors or []),
)
runtime_module.save(self.db_session)
# Update totals in runtime
runtime.count_call_functions += runtime_module.count_call_functions
runtime.count_events += runtime_module.count_events
runtime.count_storage_functions += runtime_module.count_storage_functions
runtime.count_constants += runtime_module.count_constants
runtime.count_errors += runtime_module.count_errors
if len(module.calls or []) > 0:
for idx, call in enumerate(module.calls):
runtime_call = RuntimeCall(
spec_version=spec_version,
module_id=module_id,
call_id=call.get_identifier(),
index=idx,
name=call.name,
lookup=call.lookup,
documentation='\n'.join(call.docs),
count_params=len(call.args)
)
runtime_call.save(self.db_session)
for arg in call.args:
runtime_call_param = RuntimeCallParam(
runtime_call_id=runtime_call.id,
name=arg.name,
type=arg.type
)
runtime_call_param.save(self.db_session)
if len(module.events or []) > 0:
for event_index, event in enumerate(module.events):
runtime_event = RuntimeEvent(
spec_version=spec_version,
module_id=module_id,
event_id=event.name,
index=event_index,
name=event.name,
lookup=event.lookup,
documentation='\n'.join(event.docs),
count_attributes=len(event.args)
)
runtime_event.save(self.db_session)
for arg_index, arg in enumerate(event.args):
runtime_event_attr = RuntimeEventAttribute(
runtime_event_id=runtime_event.id,
index=arg_index,
type=arg
)
runtime_event_attr.save(self.db_session)
if len(storage_functions) > 0:
for idx, storage in enumerate(storage_functions):
# Determine type
type_hasher = None
type_key1 = None
type_key2 = None
type_value = None
type_is_linked = None
type_key2hasher = None
if storage.type.get('PlainType'):
type_value = storage.type.get('PlainType')
elif storage.type.get('MapType'):
type_hasher = storage.type['MapType'].get('hasher')
type_key1 = storage.type['MapType'].get('key')
type_value = storage.type['MapType'].get('value')
type_is_linked = storage.type['MapType'].get('isLinked', False)
elif storage.type.get('DoubleMapType'):
type_hasher = storage.type['DoubleMapType'].get('hasher')
type_key1 = storage.type['DoubleMapType'].get('key1')
type_key2 = storage.type['DoubleMapType'].get('key2')
type_value = storage.type['DoubleMapType'].get('value')
type_key2hasher = storage.type['DoubleMapType'].get('key2Hasher')
runtime_storage = RuntimeStorage(
spec_version=spec_version,
module_id=module_id,
index=idx,
name=storage.name,
lookup=None,
default=storage.fallback,
modifier=storage.modifier,
type_hasher=type_hasher,
storage_key=xxh128(module.prefix.encode()) + xxh128(storage.name.encode()),
type_key1=type_key1,
type_key2=type_key2,
type_value=type_value,
type_is_linked=type_is_linked,
type_key2hasher=type_key2hasher,
documentation='\n'.join(storage.docs)
)
runtime_storage.save(self.db_session)
if len(module.constants or []) > 0:
for idx, constant in enumerate(module.constants):
# Decode value
try:
value_obj = ScaleDecoder.get_decoder_class(
constant.type,
ScaleBytes(constant.constant_value)
)
value_obj.decode()
value = value_obj.serialize()
except ValueError:
value = constant.constant_value
except RemainingScaleBytesNotEmptyException:
value = constant.constant_value
except NotImplementedError:
value = constant.constant_value
if type(value) is list or type(value) is dict:
value = json.dumps(value)
runtime_constant = RuntimeConstant(
spec_version=spec_version,
module_id=module_id,
index=idx,
name=constant.name,
type=constant.type,
value=value,
documentation='\n'.join(constant.docs)
)
runtime_constant.save(self.db_session)
if len(module.errors or []) > 0:
for idx, error in enumerate(module.errors):
runtime_error = RuntimeErrorMessage(
spec_version=spec_version,
module_id=module_id,
index=idx,
name=error.name,
documentation='\n'.join(error.docs)
)
runtime_error.save(self.db_session)
runtime.save(self.db_session)
# Process types
for runtime_type_data in list(self.substrate.get_type_registry(block_hash=block_hash).values()):
runtime_type = RuntimeType(
spec_version=runtime_type_data["spec_version"],
type_string=runtime_type_data["type_string"],
decoder_class=runtime_type_data["decoder_class"],
is_primitive_core=runtime_type_data["is_primitive_core"],
is_primitive_runtime=runtime_type_data["is_primitive_runtime"]
)
runtime_type.save(self.db_session)
self.db_session.commit()
# Put in local store
self.metadata_store[spec_version] = self.substrate.metadata_decoder
except SQLAlchemyError as e:
self.db_session.rollback()
def add_block(self, block_hash):
# Check if block is already process
if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0:
raise BlockAlreadyAdded(block_hash)
if settings.SUBSTRATE_MOCK_EXTRINSICS:
self.substrate.mock_extrinsics = settings.SUBSTRATE_MOCK_EXTRINSICS
json_block = self.substrate.get_chain_block(block_hash)
parent_hash = json_block['block']['header'].pop('parentHash')
block_id = json_block['block']['header'].pop('number')
extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot')
state_root = json_block['block']['header'].pop('stateRoot')
digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None)
# Convert block number to numeric
if not block_id.isnumeric():
block_id = int(block_id, 16)
# ==== Get block runtime from Substrate ==================
self.substrate.init_runtime(block_hash=block_hash)
self.process_metadata(self.substrate.runtime_version, block_hash)
# ==== Get parent block runtime ===================
if block_id > 0:
json_parent_runtime_version = self.substrate.get_block_runtime_version(parent_hash)
parent_spec_version = json_parent_runtime_version.get('specVersion', 0)
self.process_metadata(parent_spec_version, parent_hash)
else:
parent_spec_version = self.substrate.runtime_version
# ==== Set initial block properties =====================
block = Block(
id=block_id,
parent_id=block_id - 1,
hash=block_hash,
parent_hash=parent_hash,
state_root=state_root,
extrinsics_root=extrinsics_root,
count_extrinsics=0,
count_events=0,
count_accounts_new=0,
count_accounts_reaped=0,
count_accounts=0,
count_events_extrinsic=0,
count_events_finalization=0,
count_events_module=0,
count_events_system=0,
count_extrinsics_error=0,
count_extrinsics_signed=0,
count_extrinsics_signedby_address=0,
count_extrinsics_signedby_index=0,
count_extrinsics_success=0,
count_extrinsics_unsigned=0,
count_sessions_new=0,
count_contracts_new=0,
count_log=0,
range10000=math.floor(block_id / 10000),
range100000=math.floor(block_id / 100000),
range1000000=math.floor(block_id / 1000000),
spec_version_id=self.substrate.runtime_version,
logs=digest_logs
)
# Set temp helper variables
block._accounts_new = []
block._accounts_reaped = []
# ==== Get block events from Substrate ==================
extrinsic_success_idx = {}
events = []
try:
events_decoder = self.substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version])
event_idx = 0
for event in events_decoder.elements:
event.value['module_id'] = event.value['module_id'].lower()
model = Event(
block_id=block_id,
event_idx=event_idx,
phase=event.value['phase'],
extrinsic_idx=event.value['extrinsic_idx'],
type=event.value['type'],
spec_version_id=parent_spec_version,
module_id=event.value['module_id'],
event_id=event.value['event_id'],
system=int(event.value['module_id'] == 'system'),
module=int(event.value['module_id'] != 'system'),
attributes=event.value['params'],
codec_error=False
)
# Process event
if event.value['phase'] == 0:
block.count_events_extrinsic += 1
elif event.value['phase'] == 1:
block.count_events_finalization += 1
if event.value['module_id'] == 'system':
block.count_events_system += 1
# Store result of extrinsic
if event.value['event_id'] == 'ExtrinsicSuccess':
extrinsic_success_idx[event.value['extrinsic_idx']] = True
block.count_extrinsics_success += 1
if event.value['event_id'] == 'ExtrinsicFailed':
extrinsic_success_idx[event.value['extrinsic_idx']] = False
block.count_extrinsics_error += 1
else:
block.count_events_module += 1
model.save(self.db_session)
events.append(model)
event_idx += 1
block.count_events = len(events_decoder.elements)
except SubstrateRequestException:
block.count_events = 0
# === Extract extrinsics from block ====
extrinsics_data = json_block['block'].pop('extrinsics')
block.count_extrinsics = len(extrinsics_data)
extrinsic_idx = 0
extrinsics = []
for extrinsic in extrinsics_data:
extrinsics_decoder = ExtrinsicsDecoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
extrinsic_data = extrinsics_decoder.decode()
# Lookup result of extrinsic
extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False)
model = Extrinsic(
block_id=block_id,
extrinsic_idx=extrinsic_idx,
extrinsic_hash=extrinsics_decoder.extrinsic_hash,
extrinsic_length=extrinsic_data.get('extrinsic_length'),
extrinsic_version=extrinsic_data.get('version_info'),
signed=extrinsics_decoder.contains_transaction,
unsigned=not extrinsics_decoder.contains_transaction,
signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')),
signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')),
address_length=extrinsic_data.get('account_length'),
address=extrinsic_data.get('account_id'),
account_index=extrinsic_data.get('account_index'),
account_idx=extrinsic_data.get('account_idx'),
signature=extrinsic_data.get('signature'),
nonce=extrinsic_data.get('nonce'),
era=extrinsic_data.get('era'),
call=extrinsic_data.get('call_code'),
module_id=extrinsic_data.get('call_module'),
call_id=extrinsic_data.get('call_function'),
params=extrinsic_data.get('params'),
spec_version_id=parent_spec_version,
success=int(extrinsic_success),
error=int(not extrinsic_success),
codec_error=False
)
model.save(self.db_session)
extrinsics.append(model)
extrinsic_idx += 1
# Process extrinsic
if extrinsics_decoder.contains_transaction:
block.count_extrinsics_signed += 1
if model.signedby_address:
block.count_extrinsics_signedby_address += 1
if model.signedby_index:
block.count_extrinsics_signedby_index += 1
# Add search index for signed extrinsics
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=model.extrinsic_idx,
account_id=model.address
)
search_index.save(self.db_session)
else:
block.count_extrinsics_unsigned += 1
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id):
extrinsic_processor = processor_class(block, model, substrate=self.substrate)
extrinsic_processor.accumulation_hook(self.db_session)
extrinsic_processor.process_search_index(self.db_session)
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic,
metadata=self.metadata_store.get(block.spec_version_id),
substrate=self.substrate)
event_processor.accumulation_hook(self.db_session)
event_processor.process_search_index(self.db_session)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block, substrate=self.substrate, harvester=self)
block_processor.accumulation_hook(self.db_session)
# Debug info
if settings.DEBUG:
block.debug_info = json_block
# ==== Save data block ==================================
block.save(self.db_session)
return block
def remove_block(self, block_hash):
# Retrieve block
block = Block.query(self.db_session).filter_by(hash=block_hash).first()
# Revert event processors
for event in Event.query(self.db_session).filter_by(block_id=block.id):
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, None)
event_processor.accumulation_revert(self.db_session)
# Revert extrinsic processors
for extrinsic in Extrinsic.query(self.db_session).filter_by(block_id=block.id):
for processor_class in ProcessorRegistry().get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id):
extrinsic_processor = processor_class(block, extrinsic)
extrinsic_processor.accumulation_revert(self.db_session)
# Revert block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block)
block_processor.accumulation_revert(self.db_session)
# Delete events
for item in Event.query(self.db_session).filter_by(block_id=block.id):
self.db_session.delete(item)
# Delete extrinsics
for item in Extrinsic.query(self.db_session).filter_by(block_id=block.id):
self.db_session.delete(item)
# Delete block
self.db_session.delete(block)
def sequence_block(self, block, parent_block_data=None, parent_sequenced_block_data=None):
sequenced_block = BlockTotal(
id=block.id
)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block, sequenced_block, substrate=self.substrate)
block_processor.sequencing_hook(
self.db_session,
parent_block_data,
parent_sequenced_block_data
)
extrinsics = Extrinsic.query(self.db_session).filter_by(block_id=block.id).order_by('extrinsic_idx')
for extrinsic in extrinsics:
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id):
extrinsic_processor = processor_class(block, extrinsic, substrate=self.substrate)
extrinsic_processor.sequencing_hook(
self.db_session,
parent_block_data,
parent_sequenced_block_data
)
events = Event.query(self.db_session).filter_by(block_id=block.id).order_by('event_idx')
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic, substrate=self.substrate)
event_processor.sequencing_hook(
self.db_session,
parent_block_data,
parent_sequenced_block_data
)
sequenced_block.save(self.db_session)
return sequenced_block
def integrity_checks(self):
# 1. Check finalized head
substrate = SubstrateInterface(settings.SUBSTRATE_RPC_URL)
if settings.FINALIZATION_BY_BLOCK_CONFIRMATIONS > 0:
finalized_block_hash = substrate.get_chain_head()
finalized_block_number = max(
substrate.get_block_number(finalized_block_hash) - settings.FINALIZATION_BY_BLOCK_CONFIRMATIONS, 0
)
else:
finalized_block_hash = substrate.get_chain_finalised_head()
finalized_block_number = substrate.get_block_number(finalized_block_hash)
# 2. Check integrity head
integrity_head = Status.get_status(self.db_session, 'INTEGRITY_HEAD')
if not integrity_head.value:
# Only continue if block #1 exists
if Block.query(self.db_session).filter_by(id=1).count() == 0:
raise BlockIntegrityError('Chain not at genesis')
integrity_head.value = 0
else:
integrity_head.value = int(integrity_head.value)
start_block_id = max(integrity_head.value - 1, 0)
end_block_id = finalized_block_number
chunk_size = 1000
parent_block = None
if start_block_id < end_block_id:
# Continue integrity check
# print('== Start integrity checks from {} to {} =='.format(start_block_id, end_block_id))
for block_nr in range(start_block_id, end_block_id, chunk_size):
# TODO replace limit with filter_by block range
block_range = Block.query(self.db_session).order_by('id')[block_nr:block_nr + chunk_size]
for block in block_range:
if parent_block:
if block.id != parent_block.id + 1:
# Save integrity head if block hash of parent matches with hash in node
if parent_block.hash == substrate.get_block_hash(integrity_head.value):
integrity_head.save(self.db_session)
self.db_session.commit()
raise BlockIntegrityError('Block #{} is missing.. stopping check '.format(parent_block.id + 1))
elif block.parent_hash != parent_block.hash:
self.process_reorg_block(parent_block)
self.process_reorg_block(block)
self.remove_block(block.hash)
self.remove_block(parent_block.hash)
self.db_session.commit()
self.add_block(substrate.get_block_hash(block.id))
self.add_block(substrate.get_block_hash(parent_block.id))
self.db_session.commit()
integrity_head.value = parent_block.id - 1
# Save integrity head if block hash of parent matches with hash in node
#if parent_block.parent_hash == substrate.get_block_hash(integrity_head.value):
integrity_head.save(self.db_session)
self.db_session.commit()
raise BlockIntegrityError('Block #{} failed integrity checks, Re-adding #{}.. '.format(parent_block.id, block.id))
else:
integrity_head.value = block.id
parent_block = block
if block.id == end_block_id:
break
if parent_block:
if parent_block.hash == substrate.get_block_hash(int(integrity_head.value)):
integrity_head.save(self.db_session)
self.db_session.commit()
return {'integrity_head': integrity_head.value}
def start_sequencer(self):
integrity_status = self.integrity_checks()
self.db_session.commit()
block_nr = None
integrity_head = Status.get_status(self.db_session, 'INTEGRITY_HEAD')
if not integrity_head.value:
integrity_head.value = 0
# 3. Check sequence head
sequencer_head = self.db_session.query(func.max(BlockTotal.id)).one()[0]
if sequencer_head is None:
sequencer_head = -1
# Start sequencing process
sequencer_parent_block = BlockTotal.query(self.db_session).filter_by(id=sequencer_head).first()
parent_block = Block.query(self.db_session).filter_by(id=sequencer_head).first()
for block_nr in range(sequencer_head + 1, int(integrity_head.value) + 1):
if block_nr == 0:
# No block ever sequenced, check if chain is at genesis state
assert (not sequencer_parent_block)
block = Block.query(self.db_session).order_by('id').first()
if not block:
self.db_session.commit()
return {'error': 'Chain not at genesis'}
if block.id == 1:
# Add genesis block
block = self.add_block(block.parent_hash)
if block.id != 0:
self.db_session.commit()
return {'error': 'Chain not at genesis'}
self.process_genesis(block)
sequencer_parent_block_data = None
parent_block_data = None
else:
block_id = sequencer_parent_block.id + 1
assert (block_id == block_nr)
block = Block.query(self.db_session).get(block_nr)
if not block:
self.db_session.commit()
return {'result': 'Finished at #{}'.format(sequencer_parent_block.id)}
sequencer_parent_block_data = sequencer_parent_block.asdict()
parent_block_data = parent_block.asdict()
sequenced_block = self.sequence_block(block, parent_block_data, sequencer_parent_block_data)
self.db_session.commit()
parent_block = block
sequencer_parent_block = sequenced_block
if block_nr:
return {'result': 'Finished at #{}'.format(block_nr)}
else:
return {'result': 'Nothing to sequence'}
def process_reorg_block(self, block):
# Check if reorg already exists
if ReorgBlock.query(self.db_session).filter_by(hash=block.hash).count() == 0:
model = ReorgBlock(**block.asdict())
model.save(self.db_session)
for extrinsic in Extrinsic.query(self.db_session).filter_by(block_id=block.id):
model = ReorgExtrinsic(block_hash=block.hash, **extrinsic.asdict())
model.save(self.db_session)
for event in Event.query(self.db_session).filter_by(block_id=block.id):
model = ReorgEvent(block_hash=block.hash, **event.asdict())
model.save(self.db_session)
for log in Log.query(self.db_session).filter_by(block_id=block.id):
model = ReorgLog(block_hash=block.hash, **log.asdict())
model.save(self.db_session)
def rebuild_search_index(self):
self.db_session.execute('truncate table {}'.format(SearchIndex.__tablename__))
for block in Block.query(self.db_session).order_by('id').yield_per(1000):
extrinsic_lookup = {}
block._accounts_new = []
block._accounts_reaped = []
for extrinsic in Extrinsic.query(self.db_session).filter_by(block_id=block.id).order_by('extrinsic_idx'):
extrinsic_lookup[extrinsic.extrinsic_idx] = extrinsic
# Add search index for signed extrinsics
if extrinsic.address:
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=extrinsic.extrinsic_idx,
account_id=extrinsic.address
)
search_index.save(self.db_session)
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id):
extrinsic_processor = processor_class(block=block, extrinsic=extrinsic, substrate=self.substrate)
extrinsic_processor.process_search_index(self.db_session)
for event in Event.query(self.db_session).filter_by(block_id=block.id).order_by('event_idx'):
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsic_lookup[event.extrinsic_idx]
except (IndexError, KeyError):
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic,
metadata=self.metadata_store.get(block.spec_version_id),
substrate=self.substrate)
event_processor.process_search_index(self.db_session)
self.db_session.commit()
def create_full_balance_snaphot(self, block_id):
block_hash = self.substrate.get_block_hash(block_id)
# Determine if keys have Blake2_128Concat format so AccountId is stored in storage key
storage_method = self.substrate.get_metadata_storage_function(
module_name="System",
storage_name="Account",
block_hash=block_hash
)
if storage_method:
if storage_method.get("type_hasher_key1") == "Blake2_128Concat":
# get balances storage prefix
storage_key_prefix = self.substrate.generate_storage_hash(
storage_module='System',
storage_function='Account',
metadata_version=settings.SUBSTRATE_METADATA_VERSION
)
rpc_result = self.substrate.rpc_request(
'state_getKeys',
[storage_key_prefix, block_hash]
).get('result')
# Extract accounts from storage key
accounts = [storage_key[-64:] for storage_key in rpc_result if len(storage_key) == 162]
else:
# Retrieve accounts from database for legacy blocks
accounts = [account[0] for account in self.db_session.query(distinct(Account.id))]
for account_id in accounts:
self.create_balance_snapshot(block_id=block_id, account_id=account_id, block_hash=block_hash)
def create_balance_snapshot(self, block_id, account_id, block_hash=None):
if not block_hash:
block_hash = self.substrate.get_block_hash(block_id)
# Get balance for account
try:
account_info_data = self.substrate.get_runtime_state(
module='System',
storage_function='Account',
params=['0x{}'.format(account_id)],
block_hash=block_hash
).get('result')
# Make sure no rows inserted before processing this record
AccountInfoSnapshot.query(self.db_session).filter_by(block_id=block_id, account_id=account_id).delete()
if account_info_data:
account_info_obj = AccountInfoSnapshot(
block_id=block_id,
account_id=account_id,
account_info=account_info_data,
balance_free=account_info_data["data"]["free"],
balance_reserved=account_info_data["data"]["reserved"],
balance_total=account_info_data["data"]["free"] + account_info_data["data"]["reserved"],
nonce=account_info_data["nonce"]
)
else:
account_info_obj = AccountInfoSnapshot(
block_id=block_id,
account_id=account_id,
account_info=None,
balance_free=None,
balance_reserved=None,
balance_total=None,
nonce=None
)
account_info_obj.save(self.db_session)
except ValueError:
pass
def update_account_balances(self):
# set balances according to most recent snapshot
account_info = self.db_session.execute("""
select
a.account_id,
a.balance_total,
a.balance_free,
a.balance_reserved,
a.nonce
from
data_account_info_snapshot as a
inner join (
select
account_id, max(block_id) as max_block_id
from data_account_info_snapshot
group by account_id
) as b
on a.account_id = b.account_id and a.block_id = b.max_block_id
""")
for account_id, balance_total, balance_free, balance_reserved, nonce in account_info:
Account.query(self.db_session).filter_by(id=account_id).update(
{
Account.balance_total: balance_total,
Account.balance_free: balance_free,
Account.balance_reserved: balance_reserved,
Account.nonce: nonce,
}, synchronize_session='fetch'
)
def accumulate_block_recursive(self, block_hash, end_block_hash=None):
harvester = PolkascanHarvesterService(self.session,
type_registry=TYPE_REGISTRY)
harvester.metadata_store = self.metadata_store
# If metadata store isn't initialized yet, perform some tests
if not harvester.metadata_store:
print('Init: create entrypoints')
# Check if blocks exists
max_block_id = self.session.query(func.max(Block.id)).one()[0]
if not max_block_id:
# Speed up accumulating by creating several entry points
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
block_nr = substrate.get_block_number(block_hash)
if block_nr > 100:
for entry_point in range(0, block_nr, block_nr // 4)[1:-1]:
entry_point_hash = substrate.get_block_hash(entry_point)
accumulate_block_recursive.delay(entry_point_hash)
block = None
max_sequenced_block_id = False
add_count = 0
try:
for nr in range(0, 10):
if not block or block.id > 0:
# Process block
block = harvester.add_block(block_hash)
print('+ Added {} '.format(block_hash))
add_count += 1
self.session.commit()
# Break loop if targeted end block hash is reached
if block_hash == end_block_hash or block.id == 0:
break
# Continue with parent block hash
block_hash = block.parent_hash
# Update persistent metadata store in Celery task
self.metadata_store = harvester.metadata_store
if block_hash != end_block_hash and block and block.id > 0:
accumulate_block_recursive.delay(block.parent_hash, end_block_hash)
except BlockAlreadyAdded as e:
print('. Skipped {} '.format(block_hash))
start_sequencer.delay()
except IntegrityError as e:
print('. Skipped duplicate {} '.format(block_hash))
except Exception as exc:
print('! ERROR adding {}'.format(block_hash))
raise HarvesterCouldNotAddBlock(block_hash) from exc
return {
'result': '{} blocks added'.format(add_count),
'lastAddedBlockHash': block_hash,
'sequencerStartedFrom': max_sequenced_block_id
}
import json
from scalecodec.type_registry import load_type_registry_file
from substrateinterface import SubstrateInterface
substrate = SubstrateInterface(
url='wss://ws.sora2.soramitsu.co.jp',
type_registry_preset='default',
type_registry=load_type_registry_file('custom_types.json'),
)
block_hash = substrate.get_block_hash(block_id=184001)
print(block_hash)
# Retrieve extrinsics in block
result = substrate.get_runtime_block(block_hash=block_hash,
ignore_decoding_errors=True)
events = substrate.get_events(block_hash)
groupedEvents = {}
for event in events:
event = str(event)
eventdict = eval(event)
idx = eventdict['extrinsic_idx']
if idx in groupedEvents.keys():
groupedEvents[idx].append(eventdict)
else:
groupedEvents[idx] = [eventdict]
# print(len(result['block']['extrinsics']))
# print(len(groupedEvents))
def on_post(self, req, resp):
blockHash = None
if req.media.get('block_id'):
substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL,
address_type=SUBSTRATE_ADDRESS_TYPE,
type_registry_preset=TYPE_REGISTRY)
blockHash = substrate.get_block_hash(req.media.get('block_id'))
elif req.media.get('block_hash'):
blockHash = req.media.get('block_hash')
else:
resp.status = falcon.HTTP_BAD_REQUEST
resp.media = {
'errors': ['Either blockHash or block_id should be supplied']
}
if blockHash:
resp.status = falcon.HTTP_200
block = Block.query(
self.session).filter(Block.hash == blockHash).first()
blockTotal = BlockTotal.query(
self.session).filter(BlockTotal.id == block.id).first()
author = ss58_encode(
blockTotal.author.replace('0x', '')
) if blockTotal is not None and blockTotal.author is not None else None
if block:
blockInfo = {}
blockInfo["timestamp"] = block.datetime.strftime(
"%Y-%m-%d %H:%M:%S")
blockInfo["block_hash"] = block.hash
blockInfo["block_id"] = block.id
blockInfo["parent_id"] = block.id - 1 if block.id > 0 else 0
blockInfo["child_id"] = block.id + 1
blockInfo["parent_hash"] = block.parent_hash
blockInfo["state_root"] = block.state_root
blockInfo["extrinsic_root"] = block.extrinsics_root
blockInfo["validator"] = author
blockInfo["count_extrinsic"] = block.count_extrinsics
blockInfo["count_event"] = block.count_events
blockInfo["count_log"] = block.count_log
# blockInfo["age"] = time.mktime(block.datetime.timetuple())
blockInfo["age"] = block.datetime.strftime("%Y-%m-%d %H:%M:%S")
# 获取和区块相关的交易信息
extrinsics = Extrinsic.query(
self.session).filter(Extrinsic.block_id == block.id).all()
extrinsicsObj = [
{
"extrinsic_id":
'{}-{}'.format(block.id, extrinsic.extrinsic_idx),
"hash":
extrinsic.extrinsic_hash
if extrinsic.extrinsic_hash else None,
# "age": time.mktime(block.datetime.timetuple()),
"age":
block.datetime.strftime("%Y-%m-%d %H:%M:%S"),
"result":
extrinsic.success,
# "address": extrinsic.address if extrinsic.address else None,
# "module": extrinsic.module_id,
# "fee": None,
# "nonce": extrinsic.nonce if extrinsic.nonce else None,
# "call": extrinsic.call_id,
"operation":
'{}({})'.format(extrinsic.module_id,
extrinsic.call_id),
"params":
extrinsic.params
# "signature": extrinsic.signature if extrinsic.signature else None
} for extrinsic in extrinsics
]
# 获取和区块相关的日志信息
logs = Log.query(
self.session).filter(Log.block_id == block.id).all()
logsObj = [{
"log_id": '{}-{}'.format(block.id, log.log_idx),
"block_id": block.id,
"type": log.type,
"data": log.data['value']
} for log in logs]
# 获取和区块相关的事件信息
events = Event.query(
self.session).filter(Event.block_id == block.id).all()
eventObj = [{
"id":
'{}-{}'.format(block.id, event.event_idx),
"block_id":
block.id,
"block_hash":
block.hash,
"module_id":
event.module_id,
"event_id":
event.event_id,
"attributes":
event.attributes,
"operation":
'{}({})'.format(event.module_id, event.event_id),
"desc":
self.getEventDesc(event.module_id, event.event_id),
"hash":
self.getEventHash(block.id, event.extrinsic_idx)
} for event in events]
resp.media = {
'status': 'success',
'data': {
"block_info": blockInfo,
"extrinsics": extrinsicsObj,
"logs": logsObj,
"events": eventObj
}
}
else:
resp.status = falcon.HTTP_404
resp.media = {'result': 'Block not found'}
import json
from scalecodec.type_registry import load_type_registry_file
from substrateinterface import SubstrateInterface
substrate = SubstrateInterface(
url='wss://mof3.sora.org/',
type_registry_preset='default',
type_registry=load_type_registry_file(
'../harvester/app/type_registry/custom_types.json'),
)
block_hash = substrate.get_block_hash(block_id=3332543)
extrinsics = substrate.get_block(block_hash=block_hash)['extrinsics']
print('Extrinsincs:', json.dumps([e.value for e in extrinsics], indent=4))
events = substrate.get_events(block_hash)
print("Events:", json.dumps([e.value for e in events], indent=4))
def dealWithForks(self, shard_num, bid=None, substrate_url=None):
shard_num = str(shard_num)
if len(shard_num) == 2:
shard_num = int(shard_num[1:2])
print(shard_num)
else:
shard_num = int(shard_num)
print(
'== dealWithForks substrate_url* {} *shardnum=*{} *==start_block_num=*{}*'
.format(substrate_url, shard_num, bid))
substrate = SubstrateInterface(substrate_url)
# self.session.execute('delete from data_block where shard_num = %(shard_num)s ',shard_num=shard_num)
harvester = PolkascanHarvesterService(self.session,
type_registry=TYPE_REGISTRY)
harvester.metadata_store = self.metadata_store
try:
nr = 0
min_bid = find(bid, shard_num, substrate)
# min_bid = 5
# bid = 7
if (bid - min_bid) <= 1:
return {
'result':
'dealWithForks from {} to {} blocks check by shardnum of {}'.
format(min_bid, bid, shard_num),
'status':
'(bid - min_bid) <= 1,do nothing!'
}
self.session.query(Block).filter(Block.shard_num == shard_num,
Block.bid > min_bid,
Block.bid < bid).delete()
self.session.query(Extrinsic).filter(
Extrinsic.shard_num == shard_num, Extrinsic.block_id > min_bid,
Extrinsic.block_id < bid).delete()
self.session.query(Log).filter(Log.shard_num == shard_num,
Log.block_id > min_bid,
Log.block_id < bid).delete()
self.session.query(Event).filter(Event.shard_num == shard_num,
Event.block_id > min_bid,
Event.block_id < bid).delete()
for nr in range(min_bid + 1, bid):
blocka = harvester.add_block(substrate.get_block_hash(nr),
substrate_url)
if blocka:
print(
'== Added sucess dealWithForks substrate_url* {} *shardnum=*{} *==start_block_num=*{}*'
.format(substrate_url, shard_num, nr))
self.session.commit()
except BlockAlreadyAdded as e:
print('. dealWithForks Skipped {} '.format(nr))
except IntegrityError as e:
print('.dealWithForks Skipped duplicate {}=={} '.format(nr, e))
except Exception as exc:
print('!dealWithForks ERROR adding {}'.format(nr))
raise self.retry(exc=exc, countdown=60, max_retries=5)
return {
'result':
'dealWithForks from {} to {} blocks check by shardnum of {}'.format(
min_bid, bid, shard_num),
'dealWithForks_status':
'true'
}
class JSONRPCResource(BaseResource):
def __init__(self, cache_region):
self.cache_region = cache_region
# Check for custom types in Redis
self.substrate = None
custom_type_registry = self.cache_region.get('CUSTOM_TYPE_REGISTRY')
self.init_type_registry(custom_type_registry)
self.block_hash = None
self.metadata_decoder = None
self.runtime_version = None
self.metadata_cache = {}
self.methods = [
'rpc_methods',
'runtime_decodeScale',
'runtime_encodeScale',
'runtime_getMetadata',
'runtime_getMetadataModules',
'runtime_getMetadataCallFunctions',
'runtime_getMetadataCallFunction',
'runtime_getMetadataEvents',
'runtime_getMetadataEvent',
'runtime_getMetadataConstants',
'runtime_getMetadataConstant',
'runtime_getMetadataStorageFunctions',
'runtime_getMetadataStorageFunction',
'runtime_getMetadataErrors',
'runtime_getMetadataError',
'runtime_getState',
'runtime_getTypeRegistry',
'runtime_getType',
'runtime_getCustomTypes',
'runtime_addCustomType',
'runtime_setCustomTypes',
'runtime_removeCustomType',
'runtime_resetCustomTypes',
'runtime_getBlock',
'runtime_createSignaturePayload',
'runtime_createExternalSignerPayload',
'runtime_createExtrinsic',
'runtime_submitExtrinsic',
'runtime_getPaymentInfo',
'keypair_create',
'keypair_inspect',
'keypair_sign',
'keypair_verify'
]
def get_request_param(self, params):
try:
return params.pop(0)
except IndexError:
raise ValueError("Not enough parameters provided")
def init_type_registry(self, custom_type_registry=None):
if settings.TYPE_REGISTRY_FILE:
type_registry = load_type_registry_file(settings.TYPE_REGISTRY_FILE)
else:
type_registry = {}
if custom_type_registry:
type_registry.update(custom_type_registry)
self.substrate = SubstrateInterface(
url=settings.SUBSTRATE_RPC_URL,
ss58_format=settings.SUBSTRATE_ADDRESS_TYPE,
type_registry_preset=settings.TYPE_REGISTRY,
type_registry=custom_type_registry
)
if settings.DEBUG:
print('Custom types at init: ', custom_type_registry)
self.substrate.debug = True
def init_request(self, params=None):
if params:
self.block_hash = self.get_request_param(params)
if type(self.block_hash) is int:
self.block_hash = self.substrate.get_block_hash(self.block_hash)
def on_post(self, req, resp):
self.block_hash = None
self.metadata_decoder = None
self.runtime_version = None
self.substrate.request_id = req.media.get('id')
method = req.media.get('method')
params = req.media.get('params', [])
# Check request requirements
if not req.media.get('jsonrpc'):
resp.media = {
"error": {
"code": -32600,
"message": "Unsupported JSON-RPC protocol version"
},
"id": req.media.get('id')
}
elif not method:
resp.media = {
"error": {
"code": -32601,
"message": "Method not found"
},
"id": req.media.get('id')
}
elif method not in self.methods:
# Default pass through request to Substrate RPC
resp.media = self.substrate.rpc_request(method, params)
else:
resp.status = falcon.HTTP_200
try:
# Process methods
if method == 'runtime_getBlock':
self.init_request(params)
block = self.substrate.get_block(block_hash=self.block_hash)
if block:
block['extrinsics'] = [extrinsic.value for extrinsic in block['extrinsics']]
block['header']["digest"]["logs"] = [log.value for log in block['header']["digest"]["logs"]]
response = {
"jsonrpc": "2.0",
"result": block,
"id": req.media.get('id')
}
elif method == 'runtime_getState':
# Init params
storage_params = None
# Process params
module = self.get_request_param(params)
storage_function = self.get_request_param(params)
if params:
storage_params = self.get_request_param(params)
self.init_request(params)
# Get response
obj = self.substrate.query(
module=module,
storage_function=storage_function,
params=storage_params,
block_hash=self.block_hash
)
response = {'result': obj.value if obj else None}
elif method == 'runtime_getMetadata':
# Process params
self.init_request(params)
# Get response
response = self.substrate.get_runtime_metadata(block_hash=self.block_hash)
elif method in ['runtime_createSignaturePayload', 'runtime_createExternalSignerPayload']:
account = self.get_request_param(params)
call_module = self.get_request_param(params)
call_function = self.get_request_param(params)
call_params = self.get_request_param(params)
tip = self.get_request_param(params) or 0
era = self.get_request_param(params)
self.init_request(params)
try:
# Create call
call = self.substrate.compose_call(
call_module=call_module,
call_function=call_function,
call_params=call_params,
block_hash=self.block_hash
)
nonce = self.substrate.get_account_nonce(account) or 0
if isinstance(era, dict) and 'current' not in era and 'phase' not in era:
# Retrieve current block id
era['current'] = self.substrate.get_block_number(self.substrate.get_chain_head())
if method == 'runtime_createExternalSignerPayload':
include_call_length = True
else:
include_call_length = False
# Generate signature payload
signature_payload = self.substrate.generate_signature_payload(
call=call,
nonce=nonce,
tip=tip,
era=era,
include_call_length=include_call_length
)
response = {
"jsonrpc": "2.0",
"result": {
'signature_payload': str(signature_payload),
'nonce': nonce,
'era': era
},
"id": req.media.get('id')
}
except ValueError as e:
response = {
"jsonrpc": "2.0",
"error": {
"code": -999,
"message": str(e)
},
"id": req.media.get('id')
}
elif method in ['runtime_submitExtrinsic', 'runtime_createExtrinsic']:
account = self.get_request_param(params)
call_module = self.get_request_param(params)
call_function = self.get_request_param(params)
call_params = self.get_request_param(params)
tip = self.get_request_param(params) or 0
era = self.get_request_param(params)
crypto_type = int(self.get_request_param(params) or 1)
signature = self.get_request_param(params)
self.init_request(params)
try:
# Create call
call = self.substrate.compose_call(
call_module=call_module,
call_function=call_function,
call_params=call_params,
block_hash=self.block_hash
)
nonce = self.substrate.get_account_nonce(account) or 0
# Create keypair with only public given given in request
keypair = Keypair(ss58_address=account, crypto_type=crypto_type)
if isinstance(era, dict) and 'current' in era:
era['current'] = int(era['current'])
# Create extrinsic
extrinsic = self.substrate.create_signed_extrinsic(
call=call,
keypair=keypair,
nonce=nonce,
signature=signature,
tip=tip,
era=era
)
if method == 'runtime_createExtrinsic':
result = str(extrinsic.data)
else:
# Submit extrinsic to the node
extrinsic_result = self.substrate.submit_extrinsic(
extrinsic=extrinsic
)
result = {
"extrinsic_hash": extrinsic_result.extrinsic_hash,
"block_hash": extrinsic_result.block_hash,
"finalized": extrinsic_result.finalized,
}
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
except ValueError as e:
response = {
"jsonrpc": "2.0",
"error": {
"code": -999,
"message": str(e)
},
"id": req.media.get('id')
}
except SubstrateRequestException as e:
response = {
"jsonrpc": "2.0",
"error": e.args[0],
"id": req.media.get('id')
}
elif method == 'runtime_getPaymentInfo':
account = self.get_request_param(params)
call_module = self.get_request_param(params)
call_function = self.get_request_param(params)
call_params = self.get_request_param(params)
# Create call
call = self.substrate.compose_call(
call_module=call_module,
call_function=call_function,
call_params=call_params
)
# Create keypair with only public given given in request
keypair = Keypair(ss58_address=account)
response = {
"jsonrpc": "2.0",
"result": self.substrate.get_payment_info(call=call, keypair=keypair),
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataModules':
self.init_request(params)
response = {
"jsonrpc": "2.0",
"result": self.substrate.get_metadata_modules(block_hash=self.block_hash),
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataCallFunctions':
self.init_request(params)
call_list = self.substrate.get_metadata_call_functions(block_hash=self.block_hash)
response = {
"jsonrpc": "2.0",
"result": call_list,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataCallFunction':
param_call_module = self.get_request_param(params)
param_call_module_function = self.get_request_param(params)
self.init_request(params)
result = self.substrate.get_metadata_call_function(
module_name=param_call_module,
call_function_name=param_call_module_function,
block_hash=self.block_hash
)
response = {
"jsonrpc": "2.0",
"result": result.value,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataEvents':
self.init_request(params)
event_list = self.substrate.get_metadata_events(block_hash=self.block_hash)
response = {
"jsonrpc": "2.0",
"result": event_list,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataEvent':
param_call_module = self.get_request_param(params)
param_call_module_event = self.get_request_param(params)
self.init_request(params)
result = self.substrate.get_metadata_event(
module_name=param_call_module,
event_name=param_call_module_event,
block_hash=self.block_hash
)
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataConstants':
self.init_request(params)
constant_list = self.substrate.get_metadata_constants(block_hash=self.block_hash)
response = {
"jsonrpc": "2.0",
"result": constant_list,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataConstant':
module_name = self.get_request_param(params)
constant_name = self.get_request_param(params)
self.init_request(params)
result = self.substrate.get_metadata_constant(
module_name=module_name,
constant_name=constant_name,
block_hash=self.block_hash
)
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataStorageFunctions':
self.init_request(params)
storage_list = self.substrate.get_metadata_storage_functions(block_hash=self.block_hash)
response = {
"jsonrpc": "2.0",
"result": storage_list,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataStorageFunction':
module_name = self.get_request_param(params)
storage_name = self.get_request_param(params)
self.init_request(params)
result = self.substrate.get_metadata_storage_function(
module_name=module_name,
storage_name=storage_name,
block_hash=self.block_hash
)
response = {
"jsonrpc": "2.0",
"result": result.value,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataErrors':
self.init_request(params)
error_list = self.substrate.get_metadata_errors(block_hash=self.block_hash)
response = {
"jsonrpc": "2.0",
"result": error_list,
"id": req.media.get('id')
}
elif method == 'runtime_getMetadataError':
module_name = self.get_request_param(params)
error_name = self.get_request_param(params)
self.init_request(params)
result = self.substrate.get_metadata_error(
module_name=module_name,
error_name=error_name,
block_hash=self.block_hash
)
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
elif method == 'runtime_getTypeRegistry':
self.init_request(params)
result = self.substrate.get_type_registry(block_hash=self.block_hash)
if result:
result = list(result.values())
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
elif method == 'runtime_getType':
type_string = self.get_request_param(params)
self.init_request(params)
response = {
"jsonrpc": "2.0",
"result": self.substrate.get_type_definition(type_string, block_hash=self.block_hash),
"id": req.media.get('id')
}
elif method == 'runtime_addCustomType':
type_string = self.get_request_param(params)
type_definition = self.get_request_param(params)
# Retrieve current custom type registry
custom_type_registry = self.cache_region.get('CUSTOM_TYPE_REGISTRY')
if not custom_type_registry:
custom_type_registry = {
'types': {
}
}
custom_type_registry['types'][type_string] = type_definition
# TODO Try to decode given type definition
# Store updated custom type registry
self.cache_region.set('CUSTOM_TYPE_REGISTRY', custom_type_registry)
if settings.DEBUG:
print('Custom types updated to: ', custom_type_registry)
# Update runtime configuration
RuntimeConfiguration().update_type_registry(custom_type_registry)
response = {
"jsonrpc": "2.0",
"result": "Type registry updated",
"id": req.media.get('id')
}
elif method == 'runtime_setCustomTypes':
custom_types = self.get_request_param(params)
if type(custom_types) is not dict:
raise ValueError('custom types must be in format: {"type_string": "type_definition"}')
custom_type_registry = {
'types': custom_types
}
# Store updated custom type registry
self.cache_region.set('CUSTOM_TYPE_REGISTRY', custom_type_registry)
# Reset runtime configuration
RuntimeConfiguration().clear_type_registry()
self.init_type_registry(custom_type_registry)
if settings.DEBUG:
print('Custom types updated to: ', custom_type_registry)
response = {
"jsonrpc": "2.0",
"result": "Type registry updated",
"id": req.media.get('id')
}
elif method == 'runtime_resetCustomTypes':
custom_type_registry = None
# Store updated custom type registry
self.cache_region.set('CUSTOM_TYPE_REGISTRY', custom_type_registry)
# Reset runtime configuration
RuntimeConfiguration().clear_type_registry()
self.init_type_registry()
if settings.DEBUG:
print('Custom types cleared')
response = {
"jsonrpc": "2.0",
"result": "Custom types cleared",
"id": req.media.get('id')
}
elif method == 'runtime_removeCustomType':
type_string = self.get_request_param(params)
# Retrieve current custom type registry
custom_type_registry = self.cache_region.get('CUSTOM_TYPE_REGISTRY')
if custom_type_registry and type_string in custom_type_registry.get('types', {}):
del custom_type_registry['types'][type_string]
# Store updated custom type registry
self.cache_region.set('CUSTOM_TYPE_REGISTRY', custom_type_registry)
# Reset runtime configuration
RuntimeConfiguration().clear_type_registry()
self.init_type_registry(custom_type_registry)
result = '"{}" removed from custom type registry'.format(type_string)
else:
result = '"{}" not found in custom type registry'.format(type_string)
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
elif method == 'runtime_getCustomTypes':
custom_type_registry = self.cache_region.get('CUSTOM_TYPE_REGISTRY')
if custom_type_registry:
result = custom_type_registry.get('types')
else:
result = {}
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
elif method == 'runtime_decodeScale':
type_string = self.get_request_param(params)
scale_hex_bytes = self.get_request_param(params)
self.init_request(params)
result = self.substrate.decode_scale(
type_string=type_string,
scale_bytes=scale_hex_bytes,
block_hash=self.block_hash
)
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
elif method == 'runtime_encodeScale':
type_string = self.get_request_param(params)
value = self.get_request_param(params)
self.init_request(params)
result = self.substrate.encode_scale(
type_string=type_string,
value=value,
block_hash=self.block_hash
)
response = {
"jsonrpc": "2.0",
"result": result,
"id": req.media.get('id')
}
elif method == 'keypair_create':
word_count = self.get_request_param(params) or 0
crypto_type = int(self.get_request_param(params) or 1)
mnemonic = Keypair.generate_mnemonic(word_count)
keypair = Keypair.create_from_mnemonic(
mnemonic=mnemonic, ss58_format=settings.SUBSTRATE_ADDRESS_TYPE, crypto_type=crypto_type
)
response = {
"jsonrpc": "2.0",
"result": {
'ss58_address': keypair.ss58_address,
'public_key': keypair.public_key,
'private_key': keypair.private_key,
'mnemonic': keypair.mnemonic,
},
"id": req.media.get('id')
}
elif method == 'keypair_inspect':
mnemonic = self.get_request_param(params)
crypto_type = int(self.get_request_param(params) or 1)
keypair = Keypair.create_from_mnemonic(
mnemonic=mnemonic, ss58_format=settings.SUBSTRATE_ADDRESS_TYPE, crypto_type=crypto_type
)
response = {
"jsonrpc": "2.0",
"result": {
'ss58_address': keypair.ss58_address,
'public_key': keypair.public_key,
'private_key': keypair.private_key,
'mnemonic': keypair.mnemonic,
},
"id": req.media.get('id')
}
elif method == 'keypair_sign':
mnemonic = self.get_request_param(params)
data = self.get_request_param(params)
crypto_type = int(self.get_request_param(params) or 1)
keypair = Keypair.create_from_mnemonic(
mnemonic=mnemonic, ss58_format=settings.SUBSTRATE_ADDRESS_TYPE, crypto_type=crypto_type
)
signature = keypair.sign(data)
response = {
"jsonrpc": "2.0",
"result": {'signature': signature},
"id": req.media.get('id')
}
elif method == 'keypair_verify':
account_address = self.get_request_param(params)
data = self.get_request_param(params)
signature = self.get_request_param(params)
crypto_type = int(self.get_request_param(params) or 1)
keypair = Keypair(
ss58_address=account_address,
ss58_format=settings.SUBSTRATE_ADDRESS_TYPE,
crypto_type=crypto_type
)
result = keypair.verify(data, signature)
response = {
"jsonrpc": "2.0",
"result": {'verified': result},
"id": req.media.get('id')
}
elif method == 'rpc_methods':
response = self.substrate.rpc_request(method, params)
# Add additional implemented method
response['result']['methods'] = sorted(response['result']['methods'] + self.methods)
else:
raise NotImplementedError('Method \'{}\' not implemented yet'.format(method))
except (ValueError, NotImplementedError) as e:
response = {
"error": {
"code": -999,
"message": str(e)
},
"id": req.media.get('id')
}
except (InvalidScaleTypeValueException, RemainingScaleBytesNotEmptyException) as e:
response = {
"error": {
"code": -998,
"message": "Decoding error, given SCALE-value or type registry might be invalid "
},
"id": req.media.get('id')
}
resp.media = response
class Subtensor:
"""
Handles interactions with the subtensor chain.
"""
custom_type_registry = {
"runtime_id": 2,
"types": {
"NeuronMetadataOf": {
"type":
"struct",
"type_mapping": [["ip", "u128"], ["port", "u16"],
["ip_type", "u8"], ["uid", "u64"],
["modality", "u8"], ["hotkey", "AccountId"],
["coldkey", "AccountId"]]
}
}
}
def __init__(self,
config: 'Munch' = None,
network: str = None,
chain_endpoint: str = None):
r""" Initializes a subtensor chain interface.
Args:
config (:obj:`Munch`, `optional`):
metagraph.Metagraph.config()
network (default='akira', type=str)
The subtensor network flag. The likely choices are:
-- akira (testing network)
-- kusanagi (main network)
If this option is set it overloads subtensor.chain_endpoint with
an entry point node from that network.
chain_endpoint (default=None, type=str)
The subtensor endpoint flag. If set, overrides the --network flag.
"""
if config == None:
config = Subtensor.default_config()
config.subtensor.network = network if network != None else config.subtensor.network
config.subtensor.chain_endpoint = chain_endpoint if chain_endpoint != None else config.subtensor.chain_endpoint
Subtensor.check_config(config)
self.config = copy.deepcopy(config)
chain_endpoint = "ws://subtensor.rawatech.com:9944" if not chain_endpoint else "ws://" + chain_endpoint
# chain_endpoint = "ws://feynman.kusanagi.bittensor.com:9944" if not chain_endpoint else "ws://" + chain_endpoint
self.substrate = SubstrateInterface(
ss58_format=42,
type_registry_preset='substrate-node-template',
type_registry=self.custom_type_registry,
url=chain_endpoint)
@staticmethod
def default_config() -> Munch:
# Parses and returns a config Munch for this object.
parser = argparse.ArgumentParser()
Subtensor.add_args(parser)
config = bittensor.config.Config.to_config(parser)
return config
@staticmethod
def add_args(parser: argparse.ArgumentParser):
try:
parser.add_argument(
'--subtensor.network',
default='kusanagi',
type=str,
help='''The subtensor network flag. The likely choices are:
-- akira (testing network)
-- kusanagi (main network)
If this option is set it overloads subtensor.chain_endpoint with
an entry point node from that network.
''')
parser.add_argument(
'--subtensor.chain_endpoint',
default=None,
type=str,
help=
'''The subtensor endpoint flag. If set, overrides the --network flag.
''')
except:
pass
@staticmethod
def check_config(config: Munch):
pass
def endpoint_for_network(self, blacklist: List[str] = []) -> str:
r""" Returns a chain endpoint based on config.subtensor.network.
Returns None if there are no available endpoints.
Raises:
endpoint (str):
Websocket endpoint or None if there are none available.
"""
# Chain endpoint overrides the --network flag.
if self.config.subtensor.chain_endpoint != None:
if self.config.subtensor.chain_endpoint in blacklist:
return None
else:
return self.config.subtensor.chain_endpoint
# Else defaults to networks.
# TODO(const): this should probably make a DNS lookup.
if self.config.subtensor.network == "akira":
akira_available = [
item for item in bittensor.__akira_entrypoints__
if item not in blacklist
]
if len(akira_available) == 0:
return None
return random.choice(akira_available)
elif self.config.subtensor.network == "boltzmann":
boltzmann_available = [
item for item in bittensor.__boltzmann_entrypoints__
if item not in blacklist
]
if len(boltzmann_available) == 0:
return None
return random.choice(boltzmann_available)
elif self.config.subtensor.network == "kusanagi":
kusanagi_available = [
item for item in bittensor.__kusanagi_entrypoints__
if item not in blacklist
]
if len(kusanagi_available) == 0:
return None
return random.choice(kusanagi_available)
elif self.config.subtensor.network == "local":
local_available = [
item for item in bittensor.__local_entrypoints__
if item not in blacklist
]
if len(local_available) == 0:
return None
return random.choice(local_available)
else:
akira_available = [
item for item in bittensor.__akira_entrypoints__
if item not in blacklist
]
if len(akira_available) == 0:
return None
return random.choice(akira_available)
# def is_connected(self) -> bool:
# r""" Returns true if the connection state as a boolean.
# Raises:
# success (bool):
# True is the websocket is connected to the chain endpoint.
# """
# loop = asyncio.get_event_loop()
# loop.set_debug(enabled=True)
# return loop.run_until_complete(self.async_is_connected())
# async def async_is_connected(self) -> bool:
# r""" Returns the connection state as a boolean.
# Raises:
# success (bool):
# True is the websocket is connected to the chain endpoint.
# """
# return self.substrate.async_is_connected()
# def check_connection(self) -> bool:
# r""" Checks if substrate_old websocket backend is connected, connects if it is not.
# """
# loop = asyncio.get_event_loop()
# loop.set_debug(enabled=True)
# return loop.run_until_complete(self.async_check_connection())
# async def async_check_connection(self) -> bool:
# r""" Checks if substrate_old websocket backend is connected, connects if it is not.
# """
# if not self.async_is_connected():
# return self.async_connect()
# return True
# def connect( self, timeout: int = 10, failure = True ) -> bool:
# r""" Attempts to connect the substrate_old interface backend.
# If the connection fails, attemps another endpoint until a timeout.
# Args:
# timeout (int):
# Time to wait before subscription times out.
# failure (bool):
# This connection attempt raises an error an a failed attempt.
# Returns:
# success (bool):
# True on success.
# """
# loop = asyncio.get_event_loop()
# loop.set_debug(enabled=True)
# return loop.run_until_complete(self.async_connect(timeout, failure))
# async def async_connect( self, timeout: int = 10, failure = True ) -> bool:
# r""" Attempts to connect the substrate_old interface backend.
# If the connection fails, attemps another endpoint until a timeout.
# Args:
# timeout (int):
# Time to wait before subscription times out.
# failure (bool):
# This connection attempt raises an error an a failed attempt.
# Returns:
# success (bool):
# True on success.
# """
# start_time = time.time()
# attempted_endpoints = []
# while True:
# def connection_error_message():
# print('''
# Check that your internet connection is working and the chain endpoints are available: <cyan>{}</cyan>
# The subtensor.network should likely be one of the following choices:
# -- local - (your locally running node)
# -- akira - (testnet)
# -- kusanagi - (mainnet)
# Or you may set the endpoint manually using the --subtensor.chain_endpoint flag
# To run a local node (See: docs/running_a_validator.md) \n
# '''.format( attempted_endpoints) )
#
# # ---- Get next endpoint ----
# ws_chain_endpoint = self.endpoint_for_network( blacklist = attempted_endpoints )
# if ws_chain_endpoint == None:
# logger.error("No more endpoints available for subtensor.network: <cyan>{}</cyan>, attempted: <cyan>{}</cyan>".format(self.config.subtensor.network, attempted_endpoints))
# connection_error_message()
# if failure:
# logger.critical('Unable to connect to network:<cyan>{}</cyan>.\nMake sure your internet connection is stable and the network is properly set.'.format(self.config.subtensor.network))
# else:
# return False
# attempted_endpoints.append(ws_chain_endpoint)
#
# # --- Attempt connection ----
# if self.substrate.async_connect( ws_chain_endpoint, timeout = 5 ):
# logger.success("Connected to network:<cyan>{}</cyan> at endpoint:<cyan>{}</cyan>".format(self.config.subtensor.network, ws_chain_endpoint))
# return True
#
# # ---- Timeout ----
# elif (time.time() - start_time) > timeout:
# logger.error( "Error while connecting to network:<cyan>{}</cyan> at endpoint: <cyan>{}</cyan>".format(self.config.subtensor.network, ws_chain_endpoint))
# connection_error_message()
# if failure:
# raise RuntimeError('Unable to connect to network:<cyan>{}</cyan>.\nMake sure your internet connection is stable and the network is properly set.'.format(self.config.subtensor.network))
# else:
# return False
# async def _submit_and_check_extrinsic(
# self,
# extrinsic,
# wait_for_inclusion:bool = False,
# wait_for_finalization: bool = False,
# timeout: int = bittensor.__blocktime__ * 3
# ) -> bool:
# r""" Makes an extrinsic call to the chain, returns true if the extrinsic send was a success.
# If wait_for_inclusion or wait_for_finalization are true, the call will return true iff the
# extrinsic enters or finalizes in a block.
# Args:
# extrinsic (substrate_old extrinsic):
# Extrinsic to send to the chain.
# wait_for_inclusion (bool):
# If set, waits for the extrinsic to enter a block before returning true,
# or returns false if the extrinsic fails to enter the block within the timeout.
# wait_for_finalization (bool):
# If set, waits for the extrinsic to be finalized on the chain before returning true,
# or returns false if the extrinsic fails to be finalized within the timeout.
# timeout (int):
# Time that this call waits for either finalization of inclusion.
# Returns:
# success (bool):
# flag is true if extrinsic was finalized or uncluded in the block.
# If we did not wait for finalization / inclusion, the response is true.
# """
# # Send extrinsic
# try:
# response = self.substrate.submit_extrinsic(
# extrinsic,
# wait_for_inclusion = wait_for_inclusion,
# wait_for_finalization = wait_for_finalization,
# timeout = timeout
# )
# except SubstrateRequestException as e:
# logger.error('Extrinsic exception with error {}', e)
# return False
# except Exception as e:
# logger.error('Error submitting extrinsic with error {}', e)
# return False
#
# # Check timeout.
# if response == None:
# logger.error('Error in extrinsic: No response within timeout')
# return False
#
# # Check result.
# if not wait_for_inclusion and not wait_for_finalization:
# return True
# else:
# if 'error' in response:
# logger.error('Error in extrinsic: {}', response['error'])
# elif 'finalized' in response and response['finalized'] == True:
# return True
# elif 'inBlock' in response and response['inBlock'] == True:
# return True
# else:
# return False
def is_subscribed(self, wallet: 'bittensor.wallet.Wallet', ip: str,
port: int, modality: int) -> bool:
r""" Returns true if the bittensor endpoint is already subscribed with the wallet and metadata.
Args:
wallet (bittensor.wallet.Wallet):
bittensor wallet object.
ip (str):
endpoint host port i.e. 192.122.31.4
port (int):
endpoint port number i.e. 9221
modality (int):
int encoded endpoint modality i.e 0 for TEXT
coldkeypub (str):
string encoded coldekey pub.
"""
uid = self.get_uid_for_pubkey(wallet.hotkey.public_key)
if uid is None:
return False
neuron = self.get_neuron_for_uid(uid)
if neuron['ip'] == net.ip_to_int(ip) and neuron['port'] == port:
return True
else:
return False
def subscribe(
self,
wallet: 'bittensor.wallet.Wallet',
ip: str,
port: int,
modality: int,
wait_for_inclusion: bool = False,
wait_for_finalization=True,
timeout: int = 3 * bittensor.__blocktime__,
) -> bool:
r""" Subscribes an bittensor endpoint to the substensor chain.
Args:
wallet (bittensor.wallet.Wallet):
bittensor wallet object.
ip (str):
endpoint host port i.e. 192.122.31.4
port (int):
endpoint port number i.e. 9221
modality (int):
int encoded endpoint modality i.e 0 for TEXT
wait_for_inclusion (bool):
if set, waits for the extrinsic to enter a block before returning true,
or returns false if the extrinsic fails to enter the block within the timeout.
wait_for_finalization (bool):
if set, waits for the extrinsic to be finalized on the chain before returning true,
or returns false if the extrinsic fails to be finalized within the timeout.
timeout (int):
time that this call waits for either finalization of inclusion.
Returns:
success (bool):
flag is true if extrinsic was finalized or uncluded in the block.
If we did not wait for finalization / inclusion, the response is true.
"""
if self.is_subscribed(wallet, ip, port, modality):
logger.success(
"Already subscribed with:\n<cyan>[\n ip: {},\n port: {},\n modality: {},\n hotkey: {},\n coldkey: {}\n]</cyan>"
.format(ip, port, modality, wallet.hotkey.public_key,
wallet.coldkeypub))
return True
ip_as_int = net.ip_to_int(ip)
params = {
'ip': ip_as_int,
'port': port,
'ip_type': 4,
'modality': modality,
'coldkey': wallet.coldkeypub,
}
call = self.substrate.compose_call(call_module='SubtensorModule',
call_function='subscribe',
call_params=params)
# TODO (const): hotkey should be an argument here not assumed. Either that or the coldkey pub should also be assumed.
extrinsic = self.substrate.create_signed_extrinsic(
call=call, keypair=wallet.hotkey)
result = self.substrate.submit_extrinsic(
extrinsic, wait_for_inclusion, wait_for_finalization).is_success
if result:
logger.success(
"Successfully subscribed with:\n<cyan>[\n ip: {},\n port: {},\n modality: {},\n hotkey: {},\n coldkey: {}\n]</cyan>"
.format(ip, port, modality, wallet.hotkey.public_key,
wallet.coldkeypub))
else:
logger.error("Failed to subscribe")
return result
def add_stake(
self,
wallet: 'bittensor.wallet.Wallet',
amount: Balance,
hotkey_id: int,
wait_for_inclusion: bool = False,
wait_for_finalization: bool = False,
timeout: int = 3 * bittensor.__blocktime__,
) -> bool:
r""" Adds the specified amount of stake to passed hotkey uid.
Args:
wallet (bittensor.wallet.Wallet):
bittensor wallet object.
amount (bittensor.utils.balance.Balance):
amount to stake as bittensor balance
hotkey_id (int):
uid of hotkey to stake into.
wait_for_inclusion (bool):
if set, waits for the extrinsic to enter a block before returning true,
or returns false if the extrinsic fails to enter the block within the timeout.
wait_for_finalization (bool):
if set, waits for the extrinsic to be finalized on the chain before returning true,
or returns false if the extrinsic fails to be finalized within the timeout.
timeout (int):
time that this call waits for either finalization of inclusion.
Returns:
success (bool):
flag is true if extrinsic was finalized or uncluded in the block.
If we did not wait for finalization / inclusion, the response is true.
"""
call = self.substrate.compose_call(call_module='SubtensorModule',
call_function='add_stake',
call_params={
'hotkey': hotkey_id,
'ammount_staked': amount.rao
})
extrinsic = self.substrate.create_signed_extrinsic(
call=call, keypair=wallet.coldkey)
return self.substrate.submit_extrinsic(
extrinsic, wait_for_inclusion, wait_for_finalization).is_success
def transfer(
self,
wallet: 'bittensor.wallet.Wallet',
dest: str,
amount: Balance,
wait_for_inclusion: bool = False,
wait_for_finalization: bool = False,
timeout: int = 3 * bittensor.__blocktime__,
) -> bool:
r""" Transfers funds from this wallet to the destination public key address
Args:
wallet (bittensor.wallet.Wallet):
bittensor wallet object.
dest (str):
destination public key address of reciever.
amount (bittensor.utils.balance.Balance):
amount to stake as bittensor balance
wait_for_inclusion (bool):
if set, waits for the extrinsic to enter a block before returning true,
or returns false if the extrinsic fails to enter the block within the timeout.
wait_for_finalization (bool):
if set, waits for the extrinsic to be finalized on the chain before returning true,
or returns false if the extrinsic fails to be finalized within the timeout.
timeout (int):
time that this call waits for either finalization of inclusion.
Returns:
success (bool):
flag is true if extrinsic was finalized or uncluded in the block.
If we did not wait for finalization / inclusion, the response is true.
"""
call = self.substrate.compose_call(call_module='Balances',
call_function='transfer',
call_params={
'dest': dest,
'value': amount.rao
})
extrinsic = self.substrate.create_signed_extrinsic(
call=call, keypair=wallet.coldkey)
return self.substrate.submit_extrinsic(
extrinsic, wait_for_inclusion, wait_for_finalization).is_success
def unstake(
self,
wallet: 'bittensor.wallet.Wallet',
amount: Balance,
hotkey_id: int,
wait_for_inclusion: bool = False,
wait_for_finalization: bool = False,
timeout: int = 3 * bittensor.__blocktime__,
) -> bool:
r""" Removes stake into the wallet coldkey from the specified hotkey uid.
Args:
wallet (bittensor.wallet.Wallet):
bittensor wallet object.
amount (bittensor.utils.balance.Balance):
amount to stake as bittensor balance
hotkey_id (int):
uid of hotkey to unstake from.
wait_for_inclusion (bool):
if set, waits for the extrinsic to enter a block before returning true,
or returns false if the extrinsic fails to enter the block within the timeout.
wait_for_finalization (bool):
if set, waits for the extrinsic to be finalized on the chain before returning true,
or returns false if the extrinsic fails to be finalized within the timeout.
timeout (int):
time that this call waits for either finalization of inclusion.
Returns:
success (bool):
flag is true if extrinsic was finalized or uncluded in the block.
If we did not wait for finalization / inclusion, the response is true.
"""
call = self.substrate.compose_call(call_module='SubtensorModule',
call_function='remove_stake',
call_params={
'ammount_unstaked': amount.rao,
'hotkey': hotkey_id
})
extrinsic = self.substrate.create_signed_extrinsic(
call=call, keypair=wallet.coldkey)
return self.substrate.submit_extrinsic(
extrinsic, wait_for_inclusion, wait_for_finalization).is_success
def set_weights(self,
wallet: 'bittensor.wallet.Wallet',
destinations,
values,
wait_for_inclusion: bool = False,
wait_for_finalization: bool = False,
timeout: int = 3 * bittensor.__blocktime__) -> bool:
r""" Sets the given weights and values on chain for wallet hotkey account.
Args:
wallet (bittensor.wallet.Wallet):
bittensor wallet object.
destinations (List[int]):
uint64 uids of destination neurons.
values (List[int]):
u32 max encoded floating point weights.
wait_for_inclusion (bool):
if set, waits for the extrinsic to enter a block before returning true,
or returns false if the extrinsic fails to enter the block within the timeout.
wait_for_finalization (bool):
if set, waits for the extrinsic to be finalized on the chain before returning true,
or returns false if the extrinsic fails to be finalized within the timeout.
timeout (int):
time that this call waits for either finalization of inclusion.
Returns:
success (bool):
flag is true if extrinsic was finalized or uncluded in the block.
If we did not wait for finalization / inclusion, the response is true.
"""
call = self.substrate.compose_call(call_module='SubtensorModule',
call_function='set_weights',
call_params={
'dests': destinations,
'weights': values
})
extrinsic = self.substrate.create_signed_extrinsic(
call=call, keypair=wallet.hotkey)
return self.substrate.submit_extrinsic(
extrinsic,
wait_for_inclusion=wait_for_inclusion,
wait_for_finalization=wait_for_finalization).is_success
def get_balance(self, address: str) -> Balance:
r""" Returns the token balance for the passed ss58_address address
Args:
address (Substrate address format, default = 42):
ss58 chain address.
Return:
balance (bittensor.utils.balance.Balance):
account balance
"""
result = self.substrate.get_runtime_state(module='System',
storage_function='Account',
params=[address],
block_hash=None)
balance_info = result.get('result')
if not balance_info:
return Balance(0)
balance = balance_info['data']['free']
return Balance(balance)
def get_current_block(self) -> int:
r""" Returns the current block number on the chain.
Returns:
block_number (int):
Current chain blocknumber.
"""
return self.substrate.get_block_number(None)
def get_block_hash(self, block_nr):
return self.substrate.get_block_hash(block_nr)
def get_active(self) -> List[Tuple[str, int]]:
r""" Returns a list of (public key, uid) pairs one for each active peer on chain.
Returns:
active (List[Tuple[str, int]]):
List of active peers.
"""
result = self.substrate.iterate_map(
module='SubtensorModule',
storage_function='Active',
)
return result
def get_stake(self, hash=None) -> List[Tuple[int, int]]:
r""" Returns a list of (uid, stake) pairs one for each active peer on chain.
Returns:
stake (List[Tuple[int, int]]):
List of stake values.
"""
result = self.substrate.iterate_map(module='SubtensorModule',
storage_function='Stake',
block_hash=hash)
return result
def get_last_emit(self, hash=None) -> List[Tuple[int, int]]:
r""" Returns a list of (uid, last emit) pairs for each active peer on chain.
Returns:
last_emit (List[Tuple[int, int]]):
List of last emit values.
"""
result = self.substrate.iterate_map(module='SubtensorModule',
storage_function='LastEmit',
block_hash=hash)
return result
def get_weight_vals(self, hash=None) -> List[Tuple[int, List[int]]]:
r""" Returns a list of (uid, weight vals) pairs for each active peer on chain.
Returns:
weight_vals (List[Tuple[int, List[int]]]):
List of weight val pairs.
"""
result = self.substrate.iterate_map(module='SubtensorModule',
storage_function='WeightVals',
block_hash=hash)
return result
def get_weight_uids(self, hash=None) -> List[Tuple[int, int]]:
r""" Returns a list of (uid, weight uids) pairs for each active peer on chain.
Returns:
weight_uids (List[Tuple[int, List[int]]]):
List of weight uid pairs
"""
result = self.substrate.iterate_map(module='SubtensorModule',
storage_function='WeightUids',
block_hash=hash)
return result
def neurons(self, hash=None) -> List[Tuple[int, dict]]:
r""" Returns a list of neuron from the chain.
Returns:
neuron (List[Tuple[int, dict]]):
List of neuron objects.
"""
neurons = self.substrate.iterate_map(module='SubtensorModule',
storage_function='Neurons',
block_hash=hash)
return neurons
# def __convert_neuron(self, data) -> dict:
#
# return dict({
# 'coldkey': data['coldkey'],
# 'hotkey': data['hotkey'],
# 'ip_type': int(data['ip_type']),
# 'ip': int(data['ip']),
# 'port': int(data['port']),
# 'modality': int(data['modality']),
# 'uid': int(data['uid'])
# })
def get_uid_for_pubkey(self, pubkey=str) -> Optional[int]:
""" Returns the uid of the peer given passed public key string.
Args:
pubkey (str):
String encoded public key.
Returns:
uid (int):
uid of peer with hotkey equal to passed public key.
"""
result = self.substrate.get_runtime_state(module='SubtensorModule',
storage_function='Active',
params=[pubkey])
if result['result'] is None:
return None
return int(result['result'])
def get_neuron_for_uid(self, uid) -> dict:
""" Returns the neuron metadata of the peer with the passed uid.
Args:
uid (int):
Uid to query for metadata.
Returns:
metadata (Dict):
Dict in list form containing metadata of associated uid.
"""
result = self.substrate.get_runtime_state(module='SubtensorModule',
storage_function='Neurons',
params=[uid])
return result['result']
def get_stake_for_uid(self, uid) -> Balance:
r""" Returns the staked token amount of the peer with the passed uid.
Args:
uid (int):
Uid to query for metadata.
Returns:
stake (int):
Amount of staked token.
"""
stake = self.substrate.get_runtime_state(module='SubtensorModule',
storage_function='Stake',
params=[uid])
result = stake['result']
if not result:
return Balance(0)
return Balance(result)
def weight_uids_for_uid(self, uid) -> List[int]:
r""" Returns the weight uids of the peer with the passed uid.
Args:
uid (int):
Uid to query for metadata.
Returns:
weight_uids (List[int]):
Weight uids for passed uid.
"""
result = self.substrate.get_runtime_state(
module='SubtensorModule',
storage_function='WeightUids',
params=[uid])
return result['result']
def weight_vals_for_uid(self, uid) -> List[int]:
r""" Returns the weight vals of the peer with the passed uid.
Args:
uid (int):
Uid to query for metadata.
Returns:
weight_vals (List[int]):
Weight vals for passed uid.
"""
result = self.substrate.get_runtime_state(
module='SubtensorModule',
storage_function='WeightVals',
params=[uid])
return result['result']
def get_last_emit_data_for_uid(self, uid) -> int:
r""" Returns the last emit of the peer with the passed uid.
Args:
uid (int):
Uid to query for metadata.
Returns:
last_emit (int):
Last emit block numebr
"""
result = self.substrate.get_runtime_state(module='SubtensorModule',
storage_function='LastEmit',
params=[uid])
return result['result']
