def remove_block(self, block_hash):
# Retrieve block
block = Block.query(self.db_session).filter_by(hash=block_hash).first()
# Revert event processors
for event in Event.query(self.db_session).filter_by(block_id=block.id):
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, None)
event_processor.accumulation_revert(self.db_session)
# Revert extrinsic processors
for extrinsic in Extrinsic.query(self.db_session).filter_by(block_id=block.id):
for processor_class in ProcessorRegistry().get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id):
extrinsic_processor = processor_class(block, extrinsic)
extrinsic_processor.accumulation_revert(self.db_session)
# Revert block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block)
block_processor.accumulation_revert(self.db_session)
# Delete events
for item in Event.query(self.db_session).filter_by(block_id=block.id):
self.db_session.delete(item)
# Delete extrinsics
for item in Extrinsic.query(self.db_session).filter_by(block_id=block.id):
self.db_session.delete(item)
# Delete block
self.db_session.delete(block)
def get_item(self, item_id):
if '-' in item_id:
st = item_id.split("-")
return Event.query(self.session).filter_by(event_idx=int(st[2]),
block_id=int(st[1]),
shard_num=int(
st[0])).first()
else:
return Event.query(
self.session).filter_by(id=item_id.split('-')[0]).first()
def get_relationships(self, include_list, item):
relationships = {}
if 'extrinsics' in include_list:
relationships['extrinsics'] = Extrinsic.query(
self.session).filter_by(
block_id=item.bid,
shard_num=item.shard_num).order_by('extrinsic_idx')
if 'transactions' in include_list:
relationships['transactions'] = Extrinsic.query(
self.session).filter_by(
block_id=item.bid, signed=1,
shard_num=item.shard_num).order_by('extrinsic_idx')
if 'inherents' in include_list:
relationships['inherents'] = Extrinsic.query(self.session).filter(
Extrinsic.block_id == item.bid, Extrinsic.signed == 0,
Extrinsic.shard_num == item.shard_num,
Extrinsic.module_id != 'relay').order_by('extrinsic_idx')
if 'relay' in include_list:
relationships['relay'] = Extrinsic.query(self.session).filter(
Extrinsic.block_id == item.bid, Extrinsic.signed == 0,
Extrinsic.shard_num == item.shard_num,
Extrinsic.module_id == 'relay').order_by('extrinsic_idx')
if 'events' in include_list:
relationships['events'] = Event.query(self.session).filter_by(
block_id=item.bid, system=0,
shard_num=item.shard_num).order_by('event_idx')
if 'logs' in include_list:
relationships['logs'] = Log.query(self.session).filter_by(
block_id=item.bid,
shard_num=item.shard_num).order_by('log_idx')
return relationships
def apply_filters(self, query, params):
if params.get('filter[address]'):
if len(params.get('filter[address]')) == 64:
account_id = params.get('filter[address]')
else:
try:
account_id = ss58_decode(params.get('filter[address]'),
settings.SUBSTRATE_ADDRESS_TYPE)
except ValueError:
return query.filter(False)
search_index = SearchIndex.query(self.session).filter(
SearchIndex.index_type_id.in_([
settings.SEARCH_INDEX_BALANCETRANSFER,
settings.SEARCH_INDEX_CLAIMS_CLAIMED,
settings.SEARCH_INDEX_BALANCES_DEPOSIT,
settings.SEARCH_INDEX_STAKING_REWARD
]), SearchIndex.account_id == account_id).order_by(
SearchIndex.sorting_value.desc())
query = Event.query(self.session).filter(
tuple_(Event.block_id, Event.event_idx).in_(
[[s.block_id, s.event_idx]
for s in search_index])).order_by(Event.block_id.desc())
return query
def process_genesis(self, block):
# Set block time of parent block
child_block = Block.query(
self.db_session).filter_by(parent_hash=block.hash).first()
block.set_datetime(child_block.datetime)
# Retrieve genesis accounts
genesis_account_page_count = self.substrate.get_runtime_state(
module="Indices",
storage_function="NextEnumSet",
block_hash=block.hash).get('result', 0)
# Get Accounts on EnumSet
block.count_accounts_new = 0
block.count_accounts = 0
for enum_set_nr in range(0, genesis_account_page_count + 1):
genesis_accounts = self.substrate.get_runtime_state(
module="Indices",
storage_function="EnumSet",
params=[enum_set_nr],
block_hash=block.hash).get('result')
if genesis_accounts:
block.count_accounts_new += len(genesis_accounts)
block.count_accounts += len(genesis_accounts)
for idx, account_id in enumerate(genesis_accounts):
account_audit = AccountAudit(
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=ACCOUNT_AUDIT_TYPE_NEW)
account_audit.save(self.db_session)
account_index_id = enum_set_nr * 64 + idx
account_index_audit = AccountIndexAudit(
account_index_id=account_index_id,
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=ACCOUNT_INDEX_AUDIT_TYPE_NEW)
account_index_audit.save(self.db_session)
block.save(self.db_session)
# Create initial session
initial_session_event = NewSessionEventProcessor(block, Event(), None)
initial_session_event.add_session(db_session=self.db_session,
session_id=0)
def get_relationships(self, include_list, item):
relationships = {}
if 'events' in include_list:
relationships['events'] = Event.query(self.session).filter_by(
block_id=item.block_id,
extrinsic_idx=item.extrinsic_idx).order_by('event_idx')
return relationships
def rebuild_search_index(self):
self.db_session.execute('truncate table {}'.format(
SearchIndex.__tablename__))
for block in Block.query(
self.db_session).order_by('id').yield_per(1000):
extrinsic_lookup = {}
block._accounts_new = []
block._accounts_reaped = []
for extrinsic in Extrinsic.query(self.db_session).filter_by(
block_id=block.id).order_by('extrinsic_idx'):
extrinsic_lookup[extrinsic.extrinsic_idx] = extrinsic
# Add search index for signed extrinsics
if extrinsic.address:
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=extrinsic.extrinsic_idx,
account_id=extrinsic.address)
search_index.save(self.db_session)
# Process extrinsic processors
for processor_class in ProcessorRegistry(
).get_extrinsic_processors(extrinsic.module_id,
extrinsic.call_id):
extrinsic_processor = processor_class(
block=block,
extrinsic=extrinsic,
substrate=self.substrate)
extrinsic_processor.process_search_index(self.db_session)
for event in Event.query(self.db_session).filter_by(
block_id=block.id).order_by('event_idx'):
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsic_lookup[event.extrinsic_idx]
except (IndexError, KeyError):
extrinsic = None
for processor_class in ProcessorRegistry(
).get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(
block,
event,
extrinsic,
metadata=self.metadata_store.get(
block.spec_version_id),
substrate=self.substrate)
event_processor.process_search_index(self.db_session)
self.db_session.commit()
def sequence_block(self,
block,
parent_block_data=None,
parent_sequenced_block_data=None):
sequenced_block = BlockTotal(id=block.id)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block,
sequenced_block,
substrate=self.substrate)
block_processor.sequencing_hook(self.db_session, parent_block_data,
parent_sequenced_block_data)
extrinsics = Extrinsic.query(self.db_session).filter_by(
block_id=block.id).order_by('extrinsic_idx')
for extrinsic in extrinsics:
# Process extrinsic processors
for processor_class in ProcessorRegistry(
).get_extrinsic_processors(extrinsic.module_id, extrinsic.call_id):
extrinsic_processor = processor_class(block,
extrinsic,
substrate=self.substrate)
extrinsic_processor.sequencing_hook(
self.db_session, parent_block_data,
parent_sequenced_block_data)
events = Event.query(
self.db_session).filter_by(block_id=block.id).order_by('event_idx')
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(
event.module_id, event.event_id):
event_processor = processor_class(block,
event,
extrinsic,
substrate=self.substrate)
event_processor.sequencing_hook(self.db_session,
parent_block_data,
parent_sequenced_block_data)
sequenced_block.save(self.db_session)
return sequenced_block
def get_relationships(self, include_list, item):
relationships = {}
if 'attributes' in include_list:
relationships['attributes'] = RuntimeEventAttribute.query(self.session).filter_by(
runtime_event_id=item.id).order_by('id')
if 'recent_events' in include_list:
relationships['recent_events'] = Event.query(self.session).filter_by(
event_id=item.event_id, module_id=item.module_id).order_by(Event.block_id.desc())[:10]
return relationships
def rebuild_search_index(self, search_index_type_id):
if search_index_type_id == SEARCH_INDEX_SLASHED_ACCOUNT:
for event in Event.query(self.db_session).filter_by(
module_id='staking', event_id='Slash'):
processor = SlashEventProcessor(block=event.block, event=event)
processor.process_search_index(self.db_session)
self.db_session.commit()
elif search_index_type_id == SEARCH_INDEX_BALANCETRANSFER:
for event in Event.query(self.db_session).filter_by(
module_id='balances', event_id='Transfer'):
processor = BalancesTransferProcessor(block=event.block,
event=event)
processor.process_search_index(self.db_session)
self.db_session.commit()
else:
raise NotImplementedError(
'Not supported search index {}'.format(search_index_type_id))
def serialize_item(self, item):
data = item.serialize()
runtime_call = RuntimeCall.query(self.session).filter_by(
module_id=item.module_id,
call_id=item.call_id,
spec_version=item.spec_version_id).first()
data['attributes']['documentation'] = runtime_call.documentation
block = Block.query(self.session).get(item.block_id)
if block.datetime:
data['attributes']['datetime'] = block.datetime.replace(
tzinfo=pytz.UTC).isoformat()
else:
data['attributes']['datetime'] = None
if item.account:
data['attributes']['account'] = item.account.serialize()
if item.params:
item.params = self.check_params(item.params, item.serialize_id())
if item.error:
# Retrieve ExtrinsicFailed event
extrinsic_failed_event = Event.query(self.session).filter_by(
block_id=item.block_id, event_id='ExtrinsicFailed').first()
# Retrieve runtime error
if extrinsic_failed_event:
if 'Module' in extrinsic_failed_event.attributes[0]['value']:
error = RuntimeErrorMessage.query(self.session).filter_by(
module_index=extrinsic_failed_event.attributes[0]
['value']['Module']['index'],
index=extrinsic_failed_event.attributes[0]['value']
['Module']['error'],
spec_version=item.spec_version_id).first()
if error:
data['attributes'][
'error_message'] = error.documentation
elif 'BadOrigin' in extrinsic_failed_event.attributes[0][
'value']:
data['attributes']['error_message'] = 'Bad origin'
elif 'CannotLookup' in extrinsic_failed_event.attributes[0][
'value']:
data['attributes']['error_message'] = 'Cannot lookup'
return data
def process_reorg_block(self, block):
model = ReorgBlock(**block.asdict())
model.save(self.db_session)
for extrinsic in Extrinsic.query(
self.db_session).filter_by(block_id=block.id):
model = ReorgExtrinsic(block_hash=block.hash, **extrinsic.asdict())
model.save(self.db_session)
for event in Event.query(self.db_session).filter_by(block_id=block.id):
model = ReorgEvent(block_hash=block.hash, **event.asdict())
model.save(self.db_session)
for log in Log.query(self.db_session).filter_by(block_id=block.id):
model = ReorgLog(block_hash=block.hash, **log.asdict())
model.save(self.db_session)
def get_relationships(self, include_list, item):
relationships = {}
if 'extrinsics' in include_list:
relationships['extrinsics'] = Extrinsic.query(self.session).filter_by(block_id=item.id).order_by(
'extrinsic_idx')
if 'transactions' in include_list:
relationships['transactions'] = Extrinsic.query(self.session).filter_by(block_id=item.id, signed=1).order_by(
'extrinsic_idx')
if 'inherents' in include_list:
relationships['inherents'] = Extrinsic.query(self.session).filter_by(block_id=item.id, signed=0).order_by(
'extrinsic_idx')
if 'events' in include_list:
relationships['events'] = Event.query(self.session).filter_by(block_id=item.id, system=0).order_by(
'event_idx')
if 'logs' in include_list:
relationships['logs'] = Log.query(self.session).filter_by(block_id=item.id).order_by(
'log_idx')
return relationships
def serialize_item(self, item):
data = item.serialize()
runtime_call = RuntimeCall.query(self.session).filter_by(
module_id=item.module_id,
call_id=item.call_id,
spec_version=item.spec_version_id).first()
data['attributes']['documentation'] = runtime_call.documentation
block = Block.query(self.session).get(item.block_id)
data['attributes']['datetime'] = block.datetime.replace(
tzinfo=pytz.UTC).isoformat()
if item.account:
data['attributes']['account'] = item.account.serialize()
if item.error:
# Retrieve ExtrinsicFailed event
extrinsic_failed_event = Event.query(self.session).filter_by(
block_id=item.block_id, event_id='ExtrinsicFailed').first()
# Retrieve runtime error
if extrinsic_failed_event and 'Module' in extrinsic_failed_event.attributes[
0]['value']:
error = RuntimeErrorMessage.query(self.session).filter_by(
module_id=item.module_id,
index=extrinsic_failed_event.attributes[0]['value']
['Module']['error'],
spec_version=item.spec_version_id).first()
if error:
data['attributes']['error_message'] = error.documentation
return data
def add_block(self, block_hash):
# Check if block is already process
if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0:
raise BlockAlreadyAdded(block_hash)
if settings.SUBSTRATE_MOCK_EXTRINSICS:
self.substrate.mock_extrinsics = settings.SUBSTRATE_MOCK_EXTRINSICS
json_block = self.substrate.get_chain_block(block_hash)
parent_hash = json_block['block']['header'].pop('parentHash')
block_id = json_block['block']['header'].pop('number')
extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot')
state_root = json_block['block']['header'].pop('stateRoot')
digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None)
# Convert block number to numeric
if not block_id.isnumeric():
block_id = int(block_id, 16)
# ==== Get block runtime from Substrate ==================
self.substrate.init_runtime(block_hash=block_hash)
self.process_metadata(self.substrate.runtime_version, block_hash)
# ==== Get parent block runtime ===================
if block_id > 0:
json_parent_runtime_version = self.substrate.get_block_runtime_version(parent_hash)
parent_spec_version = json_parent_runtime_version.get('specVersion', 0)
self.process_metadata(parent_spec_version, parent_hash)
else:
parent_spec_version = self.substrate.runtime_version
# ==== Set initial block properties =====================
block = Block(
id=block_id,
parent_id=block_id - 1,
hash=block_hash,
parent_hash=parent_hash,
state_root=state_root,
extrinsics_root=extrinsics_root,
count_extrinsics=0,
count_events=0,
count_accounts_new=0,
count_accounts_reaped=0,
count_accounts=0,
count_events_extrinsic=0,
count_events_finalization=0,
count_events_module=0,
count_events_system=0,
count_extrinsics_error=0,
count_extrinsics_signed=0,
count_extrinsics_signedby_address=0,
count_extrinsics_signedby_index=0,
count_extrinsics_success=0,
count_extrinsics_unsigned=0,
count_sessions_new=0,
count_contracts_new=0,
count_log=0,
range10000=math.floor(block_id / 10000),
range100000=math.floor(block_id / 100000),
range1000000=math.floor(block_id / 1000000),
spec_version_id=self.substrate.runtime_version,
logs=digest_logs
)
# Set temp helper variables
block._accounts_new = []
block._accounts_reaped = []
# ==== Get block events from Substrate ==================
extrinsic_success_idx = {}
events = []
try:
events_decoder = self.substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version])
event_idx = 0
for event in events_decoder.elements:
event.value['module_id'] = event.value['module_id'].lower()
model = Event(
block_id=block_id,
event_idx=event_idx,
phase=event.value['phase'],
extrinsic_idx=event.value['extrinsic_idx'],
type=event.value['type'],
spec_version_id=parent_spec_version,
module_id=event.value['module_id'],
event_id=event.value['event_id'],
system=int(event.value['module_id'] == 'system'),
module=int(event.value['module_id'] != 'system'),
attributes=event.value['params'],
codec_error=False
)
# Process event
if event.value['phase'] == 0:
block.count_events_extrinsic += 1
elif event.value['phase'] == 1:
block.count_events_finalization += 1
if event.value['module_id'] == 'system':
block.count_events_system += 1
# Store result of extrinsic
if event.value['event_id'] == 'ExtrinsicSuccess':
extrinsic_success_idx[event.value['extrinsic_idx']] = True
block.count_extrinsics_success += 1
if event.value['event_id'] == 'ExtrinsicFailed':
extrinsic_success_idx[event.value['extrinsic_idx']] = False
block.count_extrinsics_error += 1
else:
block.count_events_module += 1
model.save(self.db_session)
events.append(model)
event_idx += 1
block.count_events = len(events_decoder.elements)
except SubstrateRequestException:
block.count_events = 0
# === Extract extrinsics from block ====
extrinsics_data = json_block['block'].pop('extrinsics')
block.count_extrinsics = len(extrinsics_data)
extrinsic_idx = 0
extrinsics = []
for extrinsic in extrinsics_data:
extrinsics_decoder = ExtrinsicsDecoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
extrinsic_data = extrinsics_decoder.decode()
# Lookup result of extrinsic
extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False)
model = Extrinsic(
block_id=block_id,
extrinsic_idx=extrinsic_idx,
extrinsic_hash=extrinsics_decoder.extrinsic_hash,
extrinsic_length=extrinsic_data.get('extrinsic_length'),
extrinsic_version=extrinsic_data.get('version_info'),
signed=extrinsics_decoder.contains_transaction,
unsigned=not extrinsics_decoder.contains_transaction,
signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')),
signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')),
address_length=extrinsic_data.get('account_length'),
address=extrinsic_data.get('account_id'),
account_index=extrinsic_data.get('account_index'),
account_idx=extrinsic_data.get('account_idx'),
signature=extrinsic_data.get('signature'),
nonce=extrinsic_data.get('nonce'),
era=extrinsic_data.get('era'),
call=extrinsic_data.get('call_code'),
module_id=extrinsic_data.get('call_module'),
call_id=extrinsic_data.get('call_function'),
params=extrinsic_data.get('params'),
spec_version_id=parent_spec_version,
success=int(extrinsic_success),
error=int(not extrinsic_success),
codec_error=False
)
model.save(self.db_session)
extrinsics.append(model)
extrinsic_idx += 1
# Process extrinsic
if extrinsics_decoder.contains_transaction:
block.count_extrinsics_signed += 1
if model.signedby_address:
block.count_extrinsics_signedby_address += 1
if model.signedby_index:
block.count_extrinsics_signedby_index += 1
# Add search index for signed extrinsics
search_index = SearchIndex(
index_type_id=settings.SEARCH_INDEX_SIGNED_EXTRINSIC,
block_id=block.id,
extrinsic_idx=model.extrinsic_idx,
account_id=model.address
)
search_index.save(self.db_session)
else:
block.count_extrinsics_unsigned += 1
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id):
extrinsic_processor = processor_class(block, model, substrate=self.substrate)
extrinsic_processor.accumulation_hook(self.db_session)
extrinsic_processor.process_search_index(self.db_session)
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic,
metadata=self.metadata_store.get(block.spec_version_id),
substrate=self.substrate)
event_processor.accumulation_hook(self.db_session)
event_processor.process_search_index(self.db_session)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block, substrate=self.substrate, harvester=self)
block_processor.accumulation_hook(self.db_session)
# Debug info
if settings.DEBUG:
block.debug_info = json_block
# ==== Save data block ==================================
block.save(self.db_session)
return block
def get_item(self, item_id):
return Event.query(self.session).get(item_id.split('-'))
def on_get(self, req, resp, network_id=None):
resp.status = falcon.HTTP_200
# TODO make caching more generic for custom resources
cache_key = '{}-{}'.format(req.method, req.url)
console_handler = logging.StreamHandler()
console_handler.setLevel('INFO')
logger = logging.getLogger('yee')
logger.setLevel('DEBUG')
logger.addHandler(console_handler)
# logger.info(cache_key)
response = self.cache_region.get(cache_key, self.cache_expiration_time)
if response is NO_VALUE:
best_block = Block.query(self.session).filter_by(
id=self.session.query(func.max(Block.id)).one()[0]).first()
total_signed_extrinsics = Extrinsic.query(
self.session).filter_by(signed=1).count()
total_accounts = Account.query(self.session).filter_by().count()
# total_events = Event.query(self.session).count()
event = Event.query(self.session).filter_by(
id=self.session.query(func.max(Event.id)).one()[0]).first()
if event is None:
eventid = 0
else:
eventid = event.id
if best_block:
substrate = SubstrateInterface(
SUBSTRATE_RPC_URL,
metadata_version=SUBSTRATE_METADATA_VERSION)
print(substrate.get_ShardCount())
response = self.get_jsonapi_response(data={
'type': 'networkstats',
'id': network_id,
'attributes': {
'best_block': best_block.id,
'total_signed_extrinsics': total_signed_extrinsics,
'total_events': eventid,
'total_events_module': int(best_block.id),
'total_blocks': 'N/A',
'total_accounts': total_accounts,
'total_runtimes': Runtime.query(self.session).count(),
'shard_count': int(substrate.get_ShardCount(), 16)
}
}, )
else:
response = self.get_jsonapi_response(data={
'type': 'networkstats',
'id': network_id,
'attributes': {
'best_block': 0,
'total_signed_extrinsics': 0,
'total_events': 0,
'total_events_module': 0,
'total_blocks': 'N/A',
'total_accounts': 0,
'total_runtimes': 0
}
}, )
self.cache_region.set(cache_key, response)
resp.set_header('X-Cache', 'MISS')
else:
resp.set_header('X-Cache', 'HIT')
resp.media = response
def get_query(self):
return Event.query(self.session).filter(
Event.module_id == 'balances',
Event.event_id == 'Transfer').order_by(Event.block_id.desc())
def get_item(self, item_id):
if len(item_id.split('-')) != 2:
return None
return Event.query(self.session).get(item_id.split('-'))
def get_query(self):
return Event.query(self.session).order_by(Event.block_id.desc())
def on_post(self, req, resp):
blockHash = None
if req.media.get('block_id'):
substrate = SubstrateInterface(url=SUBSTRATE_RPC_URL,
address_type=SUBSTRATE_ADDRESS_TYPE,
type_registry_preset=TYPE_REGISTRY)
blockHash = substrate.get_block_hash(req.media.get('block_id'))
elif req.media.get('block_hash'):
blockHash = req.media.get('block_hash')
else:
resp.status = falcon.HTTP_BAD_REQUEST
resp.media = {
'errors': ['Either blockHash or block_id should be supplied']
}
if blockHash:
resp.status = falcon.HTTP_200
block = Block.query(
self.session).filter(Block.hash == blockHash).first()
blockTotal = BlockTotal.query(
self.session).filter(BlockTotal.id == block.id).first()
author = ss58_encode(
blockTotal.author.replace('0x', '')
) if blockTotal is not None and blockTotal.author is not None else None
if block:
blockInfo = {}
blockInfo["timestamp"] = block.datetime.strftime(
"%Y-%m-%d %H:%M:%S")
blockInfo["block_hash"] = block.hash
blockInfo["block_id"] = block.id
blockInfo["parent_id"] = block.id - 1 if block.id > 0 else 0
blockInfo["child_id"] = block.id + 1
blockInfo["parent_hash"] = block.parent_hash
blockInfo["state_root"] = block.state_root
blockInfo["extrinsic_root"] = block.extrinsics_root
blockInfo["validator"] = author
blockInfo["count_extrinsic"] = block.count_extrinsics
blockInfo["count_event"] = block.count_events
blockInfo["count_log"] = block.count_log
# blockInfo["age"] = time.mktime(block.datetime.timetuple())
blockInfo["age"] = block.datetime.strftime("%Y-%m-%d %H:%M:%S")
# 获取和区块相关的交易信息
extrinsics = Extrinsic.query(
self.session).filter(Extrinsic.block_id == block.id).all()
extrinsicsObj = [
{
"extrinsic_id":
'{}-{}'.format(block.id, extrinsic.extrinsic_idx),
"hash":
extrinsic.extrinsic_hash
if extrinsic.extrinsic_hash else None,
# "age": time.mktime(block.datetime.timetuple()),
"age":
block.datetime.strftime("%Y-%m-%d %H:%M:%S"),
"result":
extrinsic.success,
# "address": extrinsic.address if extrinsic.address else None,
# "module": extrinsic.module_id,
# "fee": None,
# "nonce": extrinsic.nonce if extrinsic.nonce else None,
# "call": extrinsic.call_id,
"operation":
'{}({})'.format(extrinsic.module_id,
extrinsic.call_id),
"params":
extrinsic.params
# "signature": extrinsic.signature if extrinsic.signature else None
} for extrinsic in extrinsics
]
# 获取和区块相关的日志信息
logs = Log.query(
self.session).filter(Log.block_id == block.id).all()
logsObj = [{
"log_id": '{}-{}'.format(block.id, log.log_idx),
"block_id": block.id,
"type": log.type,
"data": log.data['value']
} for log in logs]
# 获取和区块相关的事件信息
events = Event.query(
self.session).filter(Event.block_id == block.id).all()
eventObj = [{
"id":
'{}-{}'.format(block.id, event.event_idx),
"block_id":
block.id,
"block_hash":
block.hash,
"module_id":
event.module_id,
"event_id":
event.event_id,
"attributes":
event.attributes,
"operation":
'{}({})'.format(event.module_id, event.event_id),
"desc":
self.getEventDesc(event.module_id, event.event_id),
"hash":
self.getEventHash(block.id, event.extrinsic_idx)
} for event in events]
resp.media = {
'status': 'success',
'data': {
"block_info": blockInfo,
"extrinsics": extrinsicsObj,
"logs": logsObj,
"events": eventObj
}
}
else:
resp.status = falcon.HTTP_404
resp.media = {'result': 'Block not found'}
def on_get(self, req, resp):
resp.status = falcon.HTTP_200
extrinsicId = req.params.get('extrinsic_id')
if extrinsicId is None:
resp.status = falcon.HTTP_BAD_REQUEST
resp.media = {'errors': ['Invalid extrinsic id']}
split = extrinsicId.split("-")
if split is None or len(split) != 2:
resp.status = falcon.HTTP_BAD_REQUEST
resp.media = {'errors': ['Invalid extrinsic id']}
blockId = split[0]
extrinsicIdx = split[1]
extrinsic = Extrinsic.extrinsic_by_id(self.session, blockId,
extrinsicIdx).first()
print(extrinsic.extrinsic_idx)
extrinsicDetailInfo = {}
if extrinsic is not None:
extrinsicDetailInfo["extrinsic_id"] = '{}-{}'.format(
extrinsic.block_id, extrinsic.extrinsic_idx)
extrinsicDetailInfo["block_id"] = extrinsic.block_id
extrinsicDetailInfo["extrinsic_idx"] = extrinsic.extrinsic_idx
extrinsicDetailInfo["hash"] = "0x{}".format(
extrinsic.extrinsic_hash) if extrinsic.extrinsic_hash else None
extrinsicDetailInfo["age"] = extrinsic.datetime.strftime(
"%Y-%m-%d %H:%M:%S")
extrinsicDetailInfo["result"] = extrinsic.success
extrinsicDetailInfo["operation"] = '{}({})'.format(
extrinsic.module_id, extrinsic.call_id)
extrinsicDetailInfo["module_id"] = extrinsic.module_id
extrinsicDetailInfo["call_id"] = extrinsic.call_id
extrinsicDetailInfo[
"module_name"] = extrinsic.module_name.capitalize()
extrinsicDetailInfo["call_name"] = extrinsic.call_name.capitalize()
extrinsicDetailInfo["call_desc"] = extrinsic.call_desc
extrinsicDetailInfo["nonce"] = int(
extrinsic.nonce) if extrinsic.nonce else None
extrinsicDetailInfo[
"signature"] = extrinsic.signature if extrinsic.signature else None
extrinsicDetailInfo["params"] = extrinsic.params
extrinsicDetailInfo["from"] = extrinsic.address.replace(
'0x', '') if extrinsic.address else None
extrinsicDetailInfo["from_addr"] = ss58_encode(
extrinsic.address.replace('0x',
'')) if extrinsic.address else None
params = json.loads(extrinsic.params)
for param in params:
name = param.get('name')
if name == 'dest':
toAddr = ss58_encode(param.get('value').replace('0x', ''))
extrinsicDetailInfo['to'] = param.get('value').replace(
'0x', '')
extrinsicDetailInfo['to_addr'] = toAddr
elif name == 'value':
coin = param.get('value') / 1000000000000 # 转换为单位 Unit
extrinsicDetailInfo['coin'] = coin
# 获取和区块相关的事件信息
events = Event.query(self.session).filter(
Event.block_id == extrinsic.block_id).all()
eventObj = [{
"id":
'{}-{}'.format(extrinsic.block_id, event.event_idx),
"block_id":
extrinsic.block_id,
"module_id":
event.module_id,
"event_id":
event.event_id,
"attributes":
event.attributes,
"operation":
'{}({})'.format(event.module_id, event.event_id),
"desc":
self.getEventDesc(event.module_id, event.event_id),
"hash":
self.getEventHash(extrinsic.block_id, event.extrinsic_idx)
} for event in events]
resp.media = {
'status': 'success',
'data': {
"extrinsic_detail": extrinsicDetailInfo,
"events": eventObj,
"event_count": len(eventObj)
}
}
def get_query(self):
return Event.query(self.session).filter(
Event.system == False).order_by(Event.block_id.desc())
def process_genesis(self, block):
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
# Set block time of parent block
child_block = Block.query(
self.db_session).filter_by(parent_hash=block.hash).first()
block.set_datetime(child_block.datetime)
# Retrieve genesis accounts
storage_call = RuntimeStorage.query(self.db_session).filter_by(
module_id='indices',
name='NextEnumSet',
spec_version=block.spec_version_id).first()
if storage_call:
genesis_account_page_count = substrate.get_storage(
block_hash=block.hash,
module="Indices",
function="NextEnumSet",
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher) or 0
# Get Accounts on EnumSet
storage_call = RuntimeStorage.query(self.db_session).filter_by(
module_id='indices',
name='EnumSet',
spec_version=block.spec_version_id).first()
if storage_call:
block.count_accounts_new = 0
block.count_accounts = 0
for enum_set_nr in range(0, genesis_account_page_count + 1):
account_index_u32 = U32()
account_index_u32.encode(enum_set_nr)
genesis_accounts = substrate.get_storage(
block_hash=block.hash,
module="Indices",
function="EnumSet",
params=account_index_u32.data.data.hex(),
return_scale_type=storage_call.get_return_type(),
hasher=storage_call.type_hasher)
if genesis_accounts:
block.count_accounts_new += len(genesis_accounts)
block.count_accounts += len(genesis_accounts)
for idx, account_id in enumerate(genesis_accounts):
account_audit = AccountAudit(
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=ACCOUNT_AUDIT_TYPE_NEW)
account_audit.save(self.db_session)
account_index_id = enum_set_nr * 64 + idx
account_index_audit = AccountIndexAudit(
account_index_id=account_index_id,
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=ACCOUNT_INDEX_AUDIT_TYPE_NEW)
account_index_audit.save(self.db_session)
block.save(self.db_session)
# Create initial session
initial_session_event = NewSessionEventProcessor(block, Event(), None)
initial_session_event.add_session(db_session=self.db_session,
session_id=0)
def process_genesis(self, block):
# Set block time of parent block
child_block = Block.query(self.db_session).filter_by(parent_hash=block.hash).first()
block.set_datetime(child_block.datetime)
# Retrieve genesis accounts
if settings.get_versioned_setting('SUBSTRATE_STORAGE_INDICES', block.spec_version_id) == 'Accounts':
# Get accounts from storage keys
storage_key_prefix = self.substrate.generate_storage_hash(
storage_module='System',
storage_function='Account',
metadata_version=settings.SUBSTRATE_METADATA_VERSION
)
rpc_result = self.substrate.rpc_request(
'state_getKeys',
[storage_key_prefix, block.hash]
).get('result')
# Extract accounts from storage key
genesis_accounts = [storage_key[-64:] for storage_key in rpc_result if len(storage_key) == 162]
for account_id in genesis_accounts:
account_audit = AccountAudit(
account_id=account_id,
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
elif settings.get_versioned_setting('SUBSTRATE_STORAGE_INDICES', block.spec_version_id) == 'EnumSet':
genesis_account_page_count = self.substrate.get_runtime_state(
module="Indices",
storage_function="NextEnumSet",
block_hash=block.hash
).get('result', 0)
# Get Accounts on EnumSet
block.count_accounts_new = 0
block.count_accounts = 0
for enum_set_nr in range(0, genesis_account_page_count + 1):
genesis_accounts = self.substrate.get_runtime_state(
module="Indices",
storage_function="EnumSet",
params=[enum_set_nr],
block_hash=block.hash
).get('result')
if genesis_accounts:
block.count_accounts_new += len(genesis_accounts)
block.count_accounts += len(genesis_accounts)
for idx, account_id in enumerate(genesis_accounts):
account_audit = AccountAudit(
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
account_index_id = enum_set_nr * 64 + idx
account_index_audit = AccountIndexAudit(
account_index_id=account_index_id,
account_id=account_id.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
type_id=settings.ACCOUNT_INDEX_AUDIT_TYPE_NEW
)
account_index_audit.save(self.db_session)
block.save(self.db_session)
# Add hardcoded account like treasury stored in settings
for account_id in settings.SUBSTRATE_TREASURY_ACCOUNTS:
account_audit = AccountAudit(
account_id=account_id,
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
data={'is_treasury': True},
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
# Check for sudo accounts
try:
# Update sudo key
sudo_key = self.substrate.get_runtime_state(
module='Sudo',
storage_function='Key',
block_hash=block.hash
).get('result')
account_audit = AccountAudit(
account_id=sudo_key.replace('0x', ''),
block_id=block.id,
extrinsic_idx=None,
event_idx=None,
data={'is_sudo': True},
type_id=settings.ACCOUNT_AUDIT_TYPE_NEW
)
account_audit.save(self.db_session)
except ValueError:
pass
# Create initial session
initial_session_event = NewSessionEventProcessor(
block=block, event=Event(), substrate=self.substrate
)
if settings.get_versioned_setting('NEW_SESSION_EVENT_HANDLER', block.spec_version_id):
initial_session_event.add_session(db_session=self.db_session, session_id=0)
else:
initial_session_event.add_session_old(db_session=self.db_session, session_id=0)
def add_block(self, block_hash):
# Check if block is already process
if Block.query(self.db_session).filter_by(hash=block_hash).count() > 0:
raise BlockAlreadyAdded(block_hash)
# Extract data from json_block
substrate = SubstrateInterface(SUBSTRATE_RPC_URL)
if SUBSTRATE_MOCK_EXTRINSICS:
substrate.mock_extrinsics = SUBSTRATE_MOCK_EXTRINSICS
json_block = substrate.get_chain_block(block_hash)
parent_hash = json_block['block']['header'].pop('parentHash')
block_id = json_block['block']['header'].pop('number')
extrinsics_root = json_block['block']['header'].pop('extrinsicsRoot')
state_root = json_block['block']['header'].pop('stateRoot')
digest_logs = json_block['block']['header'].get('digest', {}).pop('logs', None)
# Convert block number to numeric
if not block_id.isnumeric():
block_id = int(block_id, 16)
# ==== Get block runtime from Substrate ==================
json_runtime_version = substrate.get_block_runtime_version(block_hash)
# Get spec version
spec_version = json_runtime_version.get('specVersion', 0)
self.process_metadata(json_runtime_version, block_hash)
# ==== Get parent block runtime ===================
if block_id > 0:
json_parent_runtime_version = substrate.get_block_runtime_version(parent_hash)
parent_spec_version = json_parent_runtime_version.get('specVersion', 0)
self.process_metadata(json_parent_runtime_version, parent_hash)
else:
parent_spec_version = spec_version
# ==== Set initial block properties =====================
block = Block(
id=block_id,
parent_id=block_id - 1,
hash=block_hash,
parent_hash=parent_hash,
state_root=state_root,
extrinsics_root=extrinsics_root,
count_extrinsics=0,
count_events=0,
count_accounts_new=0,
count_accounts_reaped=0,
count_accounts=0,
count_events_extrinsic=0,
count_events_finalization=0,
count_events_module=0,
count_events_system=0,
count_extrinsics_error=0,
count_extrinsics_signed=0,
count_extrinsics_signedby_address=0,
count_extrinsics_signedby_index=0,
count_extrinsics_success=0,
count_extrinsics_unsigned=0,
count_sessions_new=0,
count_contracts_new=0,
count_log=0,
range10000=math.floor(block_id / 10000),
range100000=math.floor(block_id / 100000),
range1000000=math.floor(block_id / 1000000),
spec_version_id=spec_version,
logs=digest_logs
)
# Set temp helper variables
block._accounts_new = []
block._accounts_reaped = []
# ==== Get block events from Substrate ==================
extrinsic_success_idx = {}
events = []
try:
events_decoder = substrate.get_block_events(block_hash, self.metadata_store[parent_spec_version])
event_idx = 0
for event in events_decoder.elements:
event.value['module_id'] = event.value['module_id'].lower()
model = Event(
block_id=block_id,
event_idx=event_idx,
phase=event.value['phase'],
extrinsic_idx=event.value['extrinsic_idx'],
type=event.value['type'],
spec_version_id=parent_spec_version,
module_id=event.value['module_id'],
event_id=event.value['event_id'],
system=int(event.value['module_id'] == 'system'),
module=int(event.value['module_id'] != 'system'),
attributes=event.value['params'],
codec_error=False
)
# Process event
if event.value['phase'] == 0:
block.count_events_extrinsic += 1
elif event.value['phase'] == 1:
block.count_events_finalization += 1
if event.value['module_id'] == 'system':
block.count_events_system += 1
# Store result of extrinsic
if event.value['event_id'] == 'ExtrinsicSuccess':
extrinsic_success_idx[event.value['extrinsic_idx']] = True
block.count_extrinsics_success += 1
if event.value['event_id'] == 'ExtrinsicFailed':
extrinsic_success_idx[event.value['extrinsic_idx']] = False
block.count_extrinsics_error += 1
else:
block.count_events_module += 1
model.save(self.db_session)
events.append(model)
event_idx += 1
block.count_events = len(events_decoder.elements)
except SubstrateRequestException:
block.count_events = 0
# === Extract extrinsics from block ====
extrinsics_data = json_block['block'].pop('extrinsics')
block.count_extrinsics = len(extrinsics_data)
extrinsic_idx = 0
extrinsics = []
for extrinsic in extrinsics_data:
# Save to data table
if block_hash == '0x911a0bf66d5494b6b24f612b3cc14841134c6b73ab9ce02f7e012973070e5661':
# TODO TEMP fix for exception in Alexander network, remove when network is obsolete
extrinsics_decoder = ExtrinsicsBlock61181Decoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
else:
extrinsics_decoder = ExtrinsicsDecoder(
data=ScaleBytes(extrinsic),
metadata=self.metadata_store[parent_spec_version]
)
extrinsic_data = extrinsics_decoder.decode()
# Lookup result of extrinsic
extrinsic_success = extrinsic_success_idx.get(extrinsic_idx, False)
model = Extrinsic(
block_id=block_id,
extrinsic_idx=extrinsic_idx,
extrinsic_hash=extrinsics_decoder.extrinsic_hash,
extrinsic_length=extrinsic_data.get('extrinsic_length'),
extrinsic_version=extrinsic_data.get('version_info'),
signed=extrinsics_decoder.contains_transaction,
unsigned=not extrinsics_decoder.contains_transaction,
signedby_address=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_id')),
signedby_index=bool(extrinsics_decoder.contains_transaction and extrinsic_data.get('account_index')),
address_length=extrinsic_data.get('account_length'),
address=extrinsic_data.get('account_id'),
account_index=extrinsic_data.get('account_index'),
account_idx=extrinsic_data.get('account_idx'),
signature=extrinsic_data.get('signature'),
nonce=extrinsic_data.get('nonce'),
era=extrinsic_data.get('era'),
call=extrinsic_data.get('call_code'),
module_id=extrinsic_data.get('call_module'),
call_id=extrinsic_data.get('call_module_function'),
params=extrinsic_data.get('params'),
spec_version_id=parent_spec_version,
success=int(extrinsic_success),
error=int(not extrinsic_success),
codec_error=False
)
model.save(self.db_session)
if extrinsic_data.get('call_module_function') == 'transfer':
if len(extrinsic_data.get('params')) > 1:
_amount = extrinsic_data.get('params')[1].get('value')
else:
_amount = extrinsic_data.get('params')[0].get('value')
# [{'type': 'AccountId', 'value': '0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d',
# 'valueRaw': 'd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'},
# {'type': 'AccountId', 'value': '0x8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48',
# 'valueRaw': '8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48'}]
account_ids = events_decoder.value[1].get('params')[0:2]
transfer = Transfer(
block_id=block.id,
extrinsic_idx=model.extrinsic_idx,
data_extrinsic_idx=str(block_id) + '_' + str(model.extrinsic_idx),
transfer_from=ss58_encode(account_ids[0]['valueRaw']),
from_raw=account_ids[0]['valueRaw'],
transfer_to=ss58_encode(account_ids[1]['valueRaw']),
to_raw=account_ids[1]['valueRaw'],
hash=model.extrinsic_hash,
amount=decimal.Decimal(_amount),
block_timestamp=block.datetime,
module_id=extrinsic_data.get('call_module'),
success=int(extrinsic_success),
error=int(not extrinsic_success),
created_at=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
updated_at=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")#time.asctime(time.localtime(time.time()))
)
transfer.save(self.db_session)
extrinsics.append(model)
extrinsic_idx += 1
# Process extrinsic
if extrinsics_decoder.contains_transaction:
block.count_extrinsics_signed += 1
if model.signedby_address:
block.count_extrinsics_signedby_address += 1
if model.signedby_index:
block.count_extrinsics_signedby_index += 1
else:
block.count_extrinsics_unsigned += 1
# Process extrinsic processors
for processor_class in ProcessorRegistry().get_extrinsic_processors(model.module_id, model.call_id):
extrinsic_processor = processor_class(block, model)
extrinsic_processor.accumulation_hook(self.db_session)
# Process event processors
for event in events:
extrinsic = None
if event.extrinsic_idx is not None:
try:
extrinsic = extrinsics[event.extrinsic_idx]
except IndexError:
extrinsic = None
for processor_class in ProcessorRegistry().get_event_processors(event.module_id, event.event_id):
event_processor = processor_class(block, event, extrinsic,
metadata=self.metadata_store.get(block.spec_version_id))
event_processor.accumulation_hook(self.db_session)
# Process block processors
for processor_class in ProcessorRegistry().get_block_processors():
block_processor = processor_class(block)
block_processor.accumulation_hook(self.db_session)
# Debug info
if DEBUG:
block.debug_info = json_block
# ==== Save data block ==================================
block.save(self.db_session)
return block
